From a4f80caecaecaf77fc4ad3a888028160fcaf192e Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 5 Mar 2020 12:44:07 +0100
Subject: [PATCH 01/51] rest index action - 283 nofix

---
 .../elasticsearch/action/ActionModule.java    |  4 +
 .../rest/CompatibleHandlers.java              | 69 +++++++++++++++
 .../elasticsearch/rest/RestController.java    | 14 ++-
 .../org/elasticsearch/rest/RestHandler.java   |  8 ++
 .../rest/action/document/RestIndexAction.java | 85 ++++++++++++++++++-
 5 files changed, 177 insertions(+), 3 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java

diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index affbb7a41dd31..6ded4ab50f421 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -637,6 +637,10 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestIndexAction());
         registerHandler.accept(new CreateHandler());
         registerHandler.accept(new AutoIdHandler(nodesInCluster));
+        registerHandler.accept(new RestIndexAction.CompatibleRestIndexAction());
+        registerHandler.accept(new RestIndexAction.CompatibleCreateHandler());
+        registerHandler.accept(new RestIndexAction.CompatibleAutoIdHandler(nodesInCluster));
+
         registerHandler.accept(new RestGetAction());
         registerHandler.accept(new RestGetSourceAction());
         registerHandler.accept(new RestMultiGetAction(settings));
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
new file mode 100644
index 0000000000000..9260a9326c854
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
@@ -0,0 +1,69 @@
+package org.elasticsearch.rest;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.index.mapper.MapperService;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
+
+public class CompatibleHandlers {
+    private static final Logger logger = LogManager.getLogger(CompatibleHandlers.class);
+
+    /**
+     * Parameter that controls whether certain REST apis should include type names in their requests or responses.
+     * Note: Support for this parameter will be removed after the transition period to typeless APIs.
+     */
+    public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name";
+    public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false;
+
+    public static final String COMPATIBLE_HEADER = "Accept";
+    public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
+    public static final String COMPATIBLE_VERSION = "7";
+
+    public static Consumer<RestRequest> consumeParameterIncludeType(DeprecationLogger deprecationLogger) {
+        final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in create " +
+            "index requests is deprecated. The parameter will be removed in the next major version.";
+
+        return r -> {
+            if(r.hasParam(INCLUDE_TYPE_NAME_PARAMETER)){
+                deprecationLogger.deprecatedAndMaybeLog("create_index_with_types", TYPES_DEPRECATION_MESSAGE);
+                r.param(INCLUDE_TYPE_NAME_PARAMETER);
+            }
+        };
+    }
+
+    public static Consumer<RestRequest> consumeParameterType(DeprecationLogger deprecationLogger) {
+        String TYPES_DEPRECATION_MESSAGE = "[types removal] Using type as a path parameter is deprecated.";
+
+        return r -> {
+            deprecationLogger.deprecatedAndMaybeLog("create_index_with_types", TYPES_DEPRECATION_MESSAGE);
+            r.param("type");
+        };
+    }
+
+    public static boolean isV7Compatible(ToXContent.Params params) {
+        String param = params.param(COMPATIBLE_PARAMS_KEY);
+        return COMPATIBLE_VERSION.equals(param);
+    }
+
+    public static Map<String,Object> replaceTypeWithDoc(Map<String,Object> mappings){
+        Map<String, Object> newSource = new HashMap<>();
+
+        String typeName = mappings.keySet().iterator().next();
+        @SuppressWarnings("unchecked")
+        Map<String, Object> typedMappings = (Map<String, Object>) mappings.get(typeName);
+
+        newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings));
+        return typedMappings;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 0537b4d40a044..6b1efb8ff6adc 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -328,7 +328,12 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
                       return;
                   }
                 } else {
-                    dispatchRequest(request, channel, handler);
+                    if(handler.compatibilityRequired() == false //regular (not removed) handlers are always passed
+                        || CompatibleHandlers.isV7Compatible(request)) { //handlers that were registered compatible, require request to be compatible
+                        dispatchRequest(request, channel, handler);
+                    } else {
+                        handleCompatibleNotAllowed(rawPath,request.getHeaders(),channel);
+                    }
                     return;
                 }
             }
@@ -340,6 +345,13 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
         handleBadRequest(uri, requestMethod, channel);
     }
 
+    private void handleCompatibleNotAllowed(String rawPath, Map<String, List<String>> headers, RestChannel channel) throws IOException {
+        String msg = "compatible api can be only accessed with Compatible Header. path " + rawPath;
+        BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_FOUND, msg);
+
+        channel.sendResponse(bytesRestResponse);
+    }
+
     Iterator<MethodHandlers> getAllHandlers(@Nullable Map<String, String> requestParamsRef, String rawPath) {
         final Supplier<Map<String, String>> paramsSupplier;
         if (requestParamsRef == null) {
diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
index 0c06a84df62bc..829cd94c75146 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -89,6 +89,14 @@ default List<ReplacedRoute> replacedRoutes() {
         return Collections.emptyList();
     }
 
+    default List<Route> compatibleRoutes() {
+        return Collections.emptyList();
+    }
+
+    default boolean compatibilityRequired(){
+        return false;
+    }
+
     class Route {
 
         private final String path;
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index f155b09aafb42..f9d628a1a5d00 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -19,13 +19,18 @@
 
 package org.elasticsearch.rest.action.document;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
@@ -33,12 +38,23 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestIndexAction extends BaseRestHandler {
+    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " +
+        "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " +
+        "or /{index}/_create/{id}).";
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
+        LogManager.getLogger(RestIndexAction.class));
+    private static final Consumer<RestRequest> DEPRECATION_WARNING =
+        r -> deprecationLogger.deprecatedAndMaybeLog("index_with_types",TYPES_DEPRECATION_MESSAGE);
 
     @Override
     public List<Route> routes() {
@@ -52,7 +68,28 @@ public String getName() {
         return "document_index_action";
     }
 
-    public static final class CreateHandler extends RestIndexAction {
+    public static class CompatibleRestIndexAction extends  RestIndexAction{
+        @Override
+        public List<Route> routes() {
+            return List.of(
+                new Route(POST, "/{index}/{type}/{id}"),
+                new Route(PUT, "/{index}/{type}/{id}"));
+        }
+
+        @Override
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+            DEPRECATION_WARNING.accept(request);
+            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            return super.prepareRequest(request, client);
+        }
+
+        @Override
+        public boolean compatibilityRequired() {
+            return true;
+        }
+    }
+
+    public static class CreateHandler extends RestIndexAction {
 
         @Override
         public String getName() {
@@ -80,7 +117,28 @@ void validateOpType(String opType) {
         }
     }
 
-    public static final class AutoIdHandler extends RestIndexAction {
+    public static class CompatibleCreateHandler extends CreateHandler {
+        @Override
+        public List<Route> routes() {
+            return unmodifiableList(asList(
+                new Route(POST, "/{index}/{type}/{id}/_create"),
+                new Route(PUT, "/{index}/{type}/{id}/_create")));
+        }
+
+        @Override
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+            DEPRECATION_WARNING.accept(request);
+            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            return super.prepareRequest(request, client);
+        }
+
+        @Override
+        public boolean compatibilityRequired() {
+            return true;
+        }
+    }
+
+    public static class AutoIdHandler extends RestIndexAction {
 
         private final Supplier<DiscoveryNodes> nodesInCluster;
 
@@ -108,6 +166,29 @@ public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient
             return super.prepareRequest(request, client);
         }
     }
+    public static final class CompatibleAutoIdHandler extends AutoIdHandler {
+
+        public CompatibleAutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
+            super(nodesInCluster);
+        }
+
+        @Override
+        public List<Route> routes() {
+            return singletonList(new Route(POST, "/{index}/{type}"));
+        }
+
+        @Override
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+            DEPRECATION_WARNING.accept(request);
+            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            return super.prepareRequest(request, client);
+        }
+
+        @Override
+        public boolean compatibilityRequired() {
+            return true;
+        }
+    }
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {

From c99b81800e544cc45c946f3dad42a167997fd7ca Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 6 Mar 2020 11:16:50 +0100
Subject: [PATCH 02/51] Index and Get and Infra

1226 tests, 227 failures, 16ignored
23 / 2 failled in index/*
failing
CompatRestIT. test {yaml=index/70_mix_typeless_typeful/Index call that
introduces new field mappings}
CompatRestIT. test {yaml=index/70_mix_typeless_typeful/Index with
typeless API on an index that has types}
---
 .../common/xcontent/XContentBuilder.java      | 12 ++++++
 .../common/xcontent/XContentType.java         | 29 +++++++++++++-
 .../rest/compat/AbstractCompatRestTest.java   |  9 ++++-
 .../elasticsearch/action/ActionModule.java    |  2 +
 .../action/DocWriteResponse.java              |  7 ++++
 .../elasticsearch/index/get/GetResult.java    |  6 +++
 .../rest/AbstractRestChannel.java             |  2 +
 .../org/elasticsearch/rest/RestRequest.java   | 22 +++++++++-
 .../rest/action/document/RestGetAction.java   | 40 +++++++++++++++++++
 .../rest/yaml/section/MatchAssertion.java     |  6 +++
 .../security/rest/SecurityRestFilter.java     |  5 +++
 11 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 20fde0891b6f8..1dc120f9e6b10 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -165,6 +165,8 @@ public interface HumanReadableTransformer {
      */
     private boolean humanReadable = false;
 
+    private byte compatibleVersion;
+
     /**
      * Constructs a new builder using the provided XContent and an OutputStream. Make sure
      * to call {@link #close()} when the builder is done with.
@@ -998,6 +1000,16 @@ public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOExce
         return this;
     }
 
+    public XContentBuilder compatibleVersion(byte compatibleVersion){
+        this.compatibleVersion = compatibleVersion;
+        return this;
+    }
+
+    public byte getCompatibleMajorVersion() {
+        return compatibleVersion;
+    }
+
+
     @Override
     public void flush() throws IOException {
         generator.flush();
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index 606284f046244..89f9c46f38ce4 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -26,6 +26,8 @@
 
 import java.util.Locale;
 import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * The content type of {@link org.elasticsearch.common.xcontent.XContent}.
@@ -142,7 +144,9 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) {
      * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type}
      * HTTP header. This method will return {@code null} if no match is found
      */
-    public static XContentType fromMediaType(String mediaType) {
+    public static XContentType fromMediaType(String mediaTypeHeaderValue) {
+        String mediaType = parseMediaType(mediaTypeHeaderValue);
+
         final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT);
         for (XContentType type : values()) {
             if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) {
@@ -157,6 +161,29 @@ public static XContentType fromMediaType(String mediaType) {
         return null;
     }
 
+    static Pattern pattern = Pattern.compile("application/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?");
+
+    public static String parseMediaType(String mediaType) {
+        if (mediaType != null) {
+            Matcher matcher = pattern.matcher(mediaType);
+            if (matcher.find()) {
+                return "application/"+matcher.group(2);
+            }
+        }
+
+        return mediaType;
+    }
+
+    public static String parseVersion(String mediaType){
+        if(mediaType != null){
+            Matcher matcher = pattern.matcher(mediaType);
+            if (matcher.find() && "vnd.elasticsearch+".equals(matcher.group(1))) {
+
+                return matcher.group(4);
+            }
+        }
+        return null;
+    }
     private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) {
         return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) ||
                 stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") ||
diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index e465224c22e26..c2fd9a4fd33cc 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -4,6 +4,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.rest.CompatibleHandlers;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.elasticsearch.test.rest.yaml.section.DoSection;
@@ -59,10 +60,16 @@ private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
             //TODO: be more selective here
             doSection.setIgnoreWarnings(true);
             //TODO: use the real header compatibility header
-            doSection.getApiCallSection().addHeaders(Collections.singletonMap("compatible-with", "v7"));
+            String compatibleHeader = createCompatibleHeader();
+            doSection.getApiCallSection()
+                     .addHeaders(Collections.singletonMap(CompatibleHandlers.COMPATIBLE_HEADER, compatibleHeader));
         });
     }
 
+    private static String createCompatibleHeader() {
+        return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleHandlers.COMPATIBLE_VERSION;
+    }
+
 
     private static Map<ClientYamlTestCandidate, ClientYamlTestCandidate> getLocalCompatibilityTests() throws Exception {
         Iterable<Object[]> candidates =
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 6ded4ab50f421..ef68d2aa41c0b 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -642,6 +642,8 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestIndexAction.CompatibleAutoIdHandler(nodesInCluster));
 
         registerHandler.accept(new RestGetAction());
+        registerHandler.accept(new RestGetAction.CompatibleRestGetAction());
+
         registerHandler.accept(new RestGetSourceAction());
         registerHandler.accept(new RestMultiGetAction(settings));
         registerHandler.accept(new RestDeleteAction());
diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index 34f9939a366e1..ac2b8ac8e1855 100644
--- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -53,6 +54,9 @@
  */
 public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContentObject {
 
+    private static final String TYPE_FIELD_NAME = "_type";
+    private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME);
+
     private static final String _SHARDS = "_shards";
     private static final String _INDEX = "_index";
     private static final String _ID = "_id";
@@ -276,6 +280,9 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
+        if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major) {
+            builder.field(TYPE_FIELD_NAME, SINGLE_MAPPING_TYPE);
+        }
         innerToXContent(builder, params);
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index 949cc1969e7ae..a2bfb58b4dafc 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -295,11 +296,16 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
         }
         return builder;
     }
+    private static final String TYPE_FIELD_NAME = "_type";
+    private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME);
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(_INDEX, index);
+        if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major) {
+            builder.field(TYPE_FIELD_NAME, SINGLE_MAPPING_TYPE);
+        }
         builder.field(_ID, id);
         if (isExists()) {
             if (version != -1) {
diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
index 467f1d969e8be..1fe533079fff5 100644
--- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
+++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
@@ -127,6 +127,8 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu
         }
 
         builder.humanReadable(human);
+        String compatibleVersion = request.param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY);
+        builder.compatibleVersion(compatibleVersion == null ? -1 : Byte.parseByte(compatibleVersion));
         return builder;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 512bf72e9c0d3..13f1cbdf5bae3 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -131,10 +131,30 @@ void ensureSafeBuffers() {
     public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, HttpChannel httpChannel) {
         Map<String, String> params = params(httpRequest.uri());
         String path = path(httpRequest.uri());
-        return new RestRequest(xContentRegistry, params, path, httpRequest.getHeaders(), httpRequest, httpChannel,
+        RestRequest restRequest = new RestRequest(xContentRegistry, params, path, httpRequest.getHeaders(), httpRequest, httpChannel,
             requestIdGenerator.incrementAndGet());
+        addCompatibleParameter(restRequest);
+        return restRequest;
     }
 
+    private static void addCompatibleParameter(RestRequest request) {
+        if (isRequestCompatible(request)) {
+            String compatibleVersion = XContentType.parseVersion(request.header(CompatibleHandlers.COMPATIBLE_HEADER));
+            request.params().put(CompatibleHandlers.COMPATIBLE_PARAMS_KEY, compatibleVersion);
+            //use it so it won't fail request validation with unused parameter
+            request.param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY);
+        }
+    }
+
+    public static boolean isRequestCompatible(RestRequest request) {
+        return isHeaderCompatible(request.header(CompatibleHandlers.COMPATIBLE_HEADER));
+    }
+    public static boolean isHeaderCompatible(String headerValue) {
+        String version = XContentType.parseVersion(headerValue);
+        return CompatibleHandlers.COMPATIBLE_VERSION.equals(version);
+    }
+
+
     private static Map<String, String> params(final String uri) {
         final Map<String, String> params = new HashMap<>();
         int index = uri.indexOf('?');
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index 82548a505b47e..e50ebbef3eaaa 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -19,12 +19,16 @@
 
 package org.elasticsearch.rest.action.document;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestActions;
@@ -33,7 +37,11 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.List;
+import java.util.function.Consumer;
 
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
@@ -41,6 +49,12 @@
 
 public class RestGetAction extends BaseRestHandler {
 
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
+    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " +
+        "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
+    private static final Consumer<RestRequest> DEPRECATION_WARNING = r -> deprecationLogger.deprecatedAndMaybeLog("get_with_types",TYPES_DEPRECATION_MESSAGE);
+
+
     @Override
     public String getName() {
         return "document_get_action";
@@ -86,4 +100,30 @@ protected RestStatus getStatus(final GetResponse response) {
         });
     }
 
+    public static class CompatibleRestGetAction extends RestGetAction {
+        private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
+        private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " +
+            "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
+        private static final Consumer<RestRequest> DEPRECATION_WARNING = r -> deprecationLogger.deprecatedAndMaybeLog("get_with_types",TYPES_DEPRECATION_MESSAGE);
+
+
+        @Override
+        public List<Route> routes() {
+            return unmodifiableList(asList(
+                new Route(GET, "/{index}/{type}/{id}"),
+                new Route(HEAD, "/{index}/{type}/{id}")));
+        }
+
+        @Override
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+            DEPRECATION_WARNING.accept(request);
+            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            return super.prepareRequest(request, client);
+        }
+
+        @Override
+        public boolean compatibilityRequired() {
+            return true;
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
index 211fa2f20959e..b74cfd01007ef 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
@@ -56,6 +56,12 @@ public MatchAssertion(XContentLocation location, String field, Object expectedVa
 
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
+        // TODO this needs to be moved to override directory
+        if(getField().equals("_type") ){
+            assertThat(actualValue, equalTo("_doc"));
+            return;
+        }
+
         //if the value is wrapped into / it is a regexp (e.g. /s+d+/)
         if (expectedValue instanceof String) {
             String expValue = ((String) expectedValue).trim();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java
index a2b159c1024b0..6a9a38d1f7565 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java
@@ -127,4 +127,9 @@ private RestRequest maybeWrapRestRequest(RestRequest restRequest) throws IOExcep
         }
         return restRequest;
     }
+
+    @Override
+    public boolean compatibilityRequired() {
+        return restHandler.compatibilityRequired();
+    }
 }

From c9c0e55319bbc7f2f8ae6522d5347713ad1e406e Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 6 Mar 2020 16:19:50 +0100
Subject: [PATCH 03/51] minor tests

---
 .../rest/compat/AbstractCompatRestTest.java   |  2 +-
 .../action/DocWriteResponse.java              |  4 +--
 .../elasticsearch/index/get/GetResult.java    |  5 ++--
 .../action/DocWriteResponseTests.java         | 30 +++++++++++++++++++
 .../common/xcontent/XContentTypeTests.java    |  7 +++++
 .../http/DefaultRestChannelTests.java         | 10 +++++++
 6 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index c2fd9a4fd33cc..731864da1ea5b 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -59,7 +59,7 @@ private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
             DoSection doSection = (DoSection) ds;
             //TODO: be more selective here
             doSection.setIgnoreWarnings(true);
-            //TODO: use the real header compatibility header
+
             String compatibleHeader = createCompatibleHeader();
             doSection.getApiCallSection()
                      .addHeaders(Collections.singletonMap(CompatibleHandlers.COMPATIBLE_HEADER, compatibleHeader));
diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index ac2b8ac8e1855..41fb9d16b96a0 100644
--- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -54,8 +54,8 @@
  */
 public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContentObject {
 
-    private static final String TYPE_FIELD_NAME = "_type";
-    private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME);
+    static final String TYPE_FIELD_NAME = "_type";
+    static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME);
 
     private static final String _SHARDS = "_shards";
     private static final String _INDEX = "_index";
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index a2bfb58b4dafc..0e88ad37c673e 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -52,6 +52,9 @@
 
 public class GetResult implements Writeable, Iterable<DocumentField>, ToXContentObject {
 
+    private static final String TYPE_FIELD_NAME = "_type";
+    private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME);
+
     public static final String _INDEX = "_index";
     public static final String _ID = "_id";
     private static final String _VERSION = "_version";
@@ -296,8 +299,6 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
         }
         return builder;
     }
-    private static final String TYPE_FIELD_NAME = "_type";
-    private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME);
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
index bb1208bc3bba1..2bdc92ec3fd8f 100644
--- a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -107,4 +107,34 @@ public void testToXContentDoesntIncludeForcedRefreshUnlessForced() throws IOExce
             }
         }
     }
+
+    public void testTypeWhenCompatible() throws IOException {
+        DocWriteResponse response =
+            new DocWriteResponse(
+                new ShardId("index", "uuid", 0),
+                "id",
+                SequenceNumbers.UNASSIGNED_SEQ_NO,
+                17,
+                0,
+                Result.CREATED) {
+                // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
+            };
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.compatibleVersion((byte)7);
+            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+                assertThat(parser.map(), hasEntry(DocWriteResponse.TYPE_FIELD_NAME,DocWriteResponse.SINGLE_MAPPING_TYPE.toString()));
+            }
+        }
+
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.compatibleVersion((byte)6);
+            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+                assertThat(parser.map(), not(hasKey(DocWriteResponse.TYPE_FIELD_NAME)));
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
index 47a470e2cea84..a2b89b2bf542f 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
@@ -84,4 +84,11 @@ public void testFromRubbish() throws Exception {
         assertThat(XContentType.fromMediaTypeOrFormat("text/plain"), nullValue());
         assertThat(XContentType.fromMediaTypeOrFormat("gobbly;goop"), nullValue());
     }
+
+    public void testMediaType() throws Exception {
+        String mediaType = XContentType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=7");
+        assertThat(mediaType,equalTo("application/json"));
+        mediaType = XContentType.parseMediaType("application/json");
+        assertThat(mediaType,equalTo("application/json"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
index 0f82be7f23b02..b1f43f010efcf 100644
--- a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
+++ b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.http;
 
+import org.apache.http.HttpHeaders;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -36,6 +37,7 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.CompatibleHandlers;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -212,6 +214,14 @@ public void testHeadersSet() {
         assertEquals(resp.contentType(), headers.get(DefaultRestChannel.CONTENT_TYPE).get(0));
     }
 
+    public void testCompatibleParamIsSet(){
+        final TestRequest httpRequest = new TestRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/");
+        httpRequest.getHeaders().put(HttpHeaders.ACCEPT, List.of("application/vnd.elasticsearch+json;compatible-with=7"));
+        final RestRequest request = RestRequest.request(xContentRegistry(), httpRequest, httpChannel);
+
+        assertEquals("7", request.param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY));
+    }
+
     public void testCookiesSet() {
         Settings settings = Settings.builder().put(HttpTransportSettings.SETTING_HTTP_RESET_COOKIES.getKey(), true).build();
         final TestRequest httpRequest = new TestRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/");

From 258542ee492623977592838f24c9950ead6bbd60 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 6 Mar 2020 19:35:18 +0100
Subject: [PATCH 04/51] compile fixees

---
 .../rest/compat/AbstractCompatRestTest.java    | 18 ++++++++++++++++++
 .../rest/compat/CompatRestIT.java              | 18 ++++++++++++++++++
 .../rest/yaml/ESClientYamlSuiteTestCase.java   | 10 +++++++---
 3 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index 731864da1ea5b..ef5de432bef71 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package org.elasticsearch.rest.compat;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
diff --git a/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/CompatRestIT.java b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/CompatRestIT.java
index 7c94c62f3200d..0aa4201778aa8 100644
--- a/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/CompatRestIT.java
+++ b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/CompatRestIT.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package org.elasticsearch.rest.compat;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index e919c00cf4d0a..6805ed2b42068 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -185,9 +185,13 @@ public static Iterable<Object[]> createParameters() throws Exception {
         return createParameters(ExecutableSection.XCONTENT_REGISTRY, TESTS_PATH);
     }
 
-    /**
-     * Create parameters for this parameterized test.
-     */
+    public static Iterable<Object[]> createParameters(NamedXContentRegistry registry) throws Exception {
+        return createParameters(registry, TESTS_PATH);
+    }
+
+        /**
+         * Create parameters for this parameterized test.
+         */
     public static Iterable<Object[]> createParameters(NamedXContentRegistry executeableSectionRegistry, String testsPath) throws Exception {
         String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root
         Map<String, Set<Path>> yamlSuites = loadSuites(testsPath, paths);

From 0dbea8a9650e63c2a7938a156639889523097a2f Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 9 Mar 2020 14:07:47 +0100
Subject: [PATCH 05/51] disable testing conventions

---
 qa/rest-compat-tests/build.gradle | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/qa/rest-compat-tests/build.gradle b/qa/rest-compat-tests/build.gradle
index ce160b6127582..f7e242a6b0bf4 100644
--- a/qa/rest-compat-tests/build.gradle
+++ b/qa/rest-compat-tests/build.gradle
@@ -20,3 +20,6 @@ integTest.dependsOn(copyRestTestsResources)
 dependencies {
   compile project(':test:framework')
 }
+
+//fix convention not finding any tests
+testingConventions.enabled = false

From 4cf6bc1ef4a7eeec3134accc607b4a89d34e1ab7 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 10 Mar 2020 13:40:06 +0100
Subject: [PATCH 06/51] assertions and todo for header fix

---
 .../main/java/org/elasticsearch/rest/CompatibleHandlers.java  | 3 +++
 .../org/elasticsearch/rest/action/document/RestGetAction.java | 4 ++++
 .../elasticsearch/rest/action/document/RestIndexAction.java   | 2 ++
 3 files changed, 9 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
index 9260a9326c854..5a454f3af9fb5 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
@@ -25,6 +25,9 @@ public class CompatibleHandlers {
     public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name";
     public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false;
 
+    /**
+     * TODO revisit when https://github.com/elastic/elasticsearch/issues/52370 is resolved
+     */
     public static final String COMPATIBLE_HEADER = "Accept";
     public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
     public static final String COMPATIBLE_VERSION = "7";
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index e50ebbef3eaaa..a5b132c2556d8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest.action.document;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
@@ -101,6 +102,7 @@ protected RestStatus getStatus(final GetResponse response) {
     }
 
     public static class CompatibleRestGetAction extends RestGetAction {
+
         private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
         private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " +
             "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
@@ -109,6 +111,8 @@ public static class CompatibleRestGetAction extends RestGetAction {
 
         @Override
         public List<Route> routes() {
+            assert Version.CURRENT.major == 8 : "REST API compatilbity for version 7 is only supported on version 8";
+
             return unmodifiableList(asList(
                 new Route(GET, "/{index}/{type}/{id}"),
                 new Route(HEAD, "/{index}/{type}/{id}")));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index f9d628a1a5d00..7f5c105cd71a0 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -71,6 +71,8 @@ public String getName() {
     public static class CompatibleRestIndexAction extends  RestIndexAction{
         @Override
         public List<Route> routes() {
+            assert Version.CURRENT.major == 8 : "REST API compatilbity for version 7 is only supported on version 8";
+
             return List.of(
                 new Route(POST, "/{index}/{type}/{id}"),
                 new Route(PUT, "/{index}/{type}/{id}"));

From 3ac22b1eeb2deb05f63dbd197255e48c927bf0a5 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 10 Mar 2020 20:18:17 +0100
Subject: [PATCH 07/51] more tests and cleanup

---
 .../common/xcontent/XContentBuilder.java      |  8 ++--
 qa/rest-compat-tests/build.gradle             |  3 --
 .../elasticsearch/action/ActionModule.java    |  2 -
 .../rest/AbstractRestChannel.java             |  2 +-
 .../rest/CompatibleHandlers.java              | 41 -------------------
 .../elasticsearch/rest/RestController.java    |  8 ++--
 .../org/elasticsearch/rest/RestHandler.java   |  4 --
 .../org/elasticsearch/rest/RestRequest.java   | 19 +++++----
 .../action/DocWriteResponseTests.java         |  4 +-
 .../rest/RestControllerTests.java             | 33 +++++++++++++++
 10 files changed, 54 insertions(+), 70 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 1dc120f9e6b10..c35f59c53cd6d 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -165,7 +165,7 @@ public interface HumanReadableTransformer {
      */
     private boolean humanReadable = false;
 
-    private byte compatibleVersion;
+    private byte compatibleMajorVersion;
 
     /**
      * Constructs a new builder using the provided XContent and an OutputStream. Make sure
@@ -1000,13 +1000,13 @@ public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOExce
         return this;
     }
 
-    public XContentBuilder compatibleVersion(byte compatibleVersion){
-        this.compatibleVersion = compatibleVersion;
+    public XContentBuilder setCompatibleMajorVersion(byte compatibleMajorVersion){
+        this.compatibleMajorVersion = compatibleMajorVersion;
         return this;
     }
 
     public byte getCompatibleMajorVersion() {
-        return compatibleVersion;
+        return compatibleMajorVersion;
     }
 
 
diff --git a/qa/rest-compat-tests/build.gradle b/qa/rest-compat-tests/build.gradle
index f7e242a6b0bf4..ce160b6127582 100644
--- a/qa/rest-compat-tests/build.gradle
+++ b/qa/rest-compat-tests/build.gradle
@@ -20,6 +20,3 @@ integTest.dependsOn(copyRestTestsResources)
 dependencies {
   compile project(':test:framework')
 }
-
-//fix convention not finding any tests
-testingConventions.enabled = false
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index ef68d2aa41c0b..0c28c51528ab3 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -640,10 +640,8 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestIndexAction.CompatibleRestIndexAction());
         registerHandler.accept(new RestIndexAction.CompatibleCreateHandler());
         registerHandler.accept(new RestIndexAction.CompatibleAutoIdHandler(nodesInCluster));
-
         registerHandler.accept(new RestGetAction());
         registerHandler.accept(new RestGetAction.CompatibleRestGetAction());
-
         registerHandler.accept(new RestGetSourceAction());
         registerHandler.accept(new RestMultiGetAction(settings));
         registerHandler.accept(new RestDeleteAction());
diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
index 1fe533079fff5..5beaa6fd8eaf2 100644
--- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
+++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
@@ -128,7 +128,7 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu
 
         builder.humanReadable(human);
         String compatibleVersion = request.param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY);
-        builder.compatibleVersion(compatibleVersion == null ? -1 : Byte.parseByte(compatibleVersion));
+        builder.setCompatibleMajorVersion(compatibleVersion == null ? -1 : Byte.parseByte(compatibleVersion));
         return builder;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
index 5a454f3af9fb5..09020e794ed8a 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
@@ -1,29 +1,11 @@
 package org.elasticsearch.rest;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
-import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.index.mapper.MapperService;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.function.Consumer;
-import java.util.function.UnaryOperator;
 
 public class CompatibleHandlers {
-    private static final Logger logger = LogManager.getLogger(CompatibleHandlers.class);
-
-    /**
-     * Parameter that controls whether certain REST apis should include type names in their requests or responses.
-     * Note: Support for this parameter will be removed after the transition period to typeless APIs.
-     */
-    public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name";
-    public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false;
 
     /**
      * TODO revisit when https://github.com/elastic/elasticsearch/issues/52370 is resolved
@@ -32,18 +14,6 @@ public class CompatibleHandlers {
     public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
     public static final String COMPATIBLE_VERSION = "7";
 
-    public static Consumer<RestRequest> consumeParameterIncludeType(DeprecationLogger deprecationLogger) {
-        final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in create " +
-            "index requests is deprecated. The parameter will be removed in the next major version.";
-
-        return r -> {
-            if(r.hasParam(INCLUDE_TYPE_NAME_PARAMETER)){
-                deprecationLogger.deprecatedAndMaybeLog("create_index_with_types", TYPES_DEPRECATION_MESSAGE);
-                r.param(INCLUDE_TYPE_NAME_PARAMETER);
-            }
-        };
-    }
-
     public static Consumer<RestRequest> consumeParameterType(DeprecationLogger deprecationLogger) {
         String TYPES_DEPRECATION_MESSAGE = "[types removal] Using type as a path parameter is deprecated.";
 
@@ -58,15 +28,4 @@ public static boolean isV7Compatible(ToXContent.Params params) {
         return COMPATIBLE_VERSION.equals(param);
     }
 
-    public static Map<String,Object> replaceTypeWithDoc(Map<String,Object> mappings){
-        Map<String, Object> newSource = new HashMap<>();
-
-        String typeName = mappings.keySet().iterator().next();
-        @SuppressWarnings("unchecked")
-        Map<String, Object> typedMappings = (Map<String, Object>) mappings.get(typeName);
-
-        newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings));
-        return typedMappings;
-    }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 6b1efb8ff6adc..06f3588138d03 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -328,11 +328,11 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
                       return;
                   }
                 } else {
-                    if(handler.compatibilityRequired() == false //regular (not removed) handlers are always passed
+                    if(handler.compatibilityRequired() == false //regular (not removed) handlers are always dispatched
                         || CompatibleHandlers.isV7Compatible(request)) { //handlers that were registered compatible, require request to be compatible
                         dispatchRequest(request, channel, handler);
                     } else {
-                        handleCompatibleNotAllowed(rawPath,request.getHeaders(),channel);
+                        handleCompatibleNotAllowed(rawPath, channel);
                     }
                     return;
                 }
@@ -345,8 +345,8 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
         handleBadRequest(uri, requestMethod, channel);
     }
 
-    private void handleCompatibleNotAllowed(String rawPath, Map<String, List<String>> headers, RestChannel channel) throws IOException {
-        String msg = "compatible api can be only accessed with Compatible Header. path " + rawPath;
+    private void handleCompatibleNotAllowed(String rawPath, RestChannel channel) throws IOException {
+        String msg = "Compatible api can be only accessed with Compatible Header. Path used: " + rawPath;
         BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_FOUND, msg);
 
         channel.sendResponse(bytesRestResponse);
diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
index 829cd94c75146..58c452897308e 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -89,10 +89,6 @@ default List<ReplacedRoute> replacedRoutes() {
         return Collections.emptyList();
     }
 
-    default List<Route> compatibleRoutes() {
-        return Collections.emptyList();
-    }
-
     default boolean compatibilityRequired(){
         return false;
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 13f1cbdf5bae3..b54f67c42f589 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -102,6 +102,7 @@ private RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String>
         this.rawPath = path;
         this.headers = Collections.unmodifiableMap(headers);
         this.requestId = requestId;
+        addCompatibleParameter();
     }
 
     protected RestRequest(RestRequest restRequest) {
@@ -133,23 +134,23 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
         String path = path(httpRequest.uri());
         RestRequest restRequest = new RestRequest(xContentRegistry, params, path, httpRequest.getHeaders(), httpRequest, httpChannel,
             requestIdGenerator.incrementAndGet());
-        addCompatibleParameter(restRequest);
         return restRequest;
     }
 
-    private static void addCompatibleParameter(RestRequest request) {
-        if (isRequestCompatible(request)) {
-            String compatibleVersion = XContentType.parseVersion(request.header(CompatibleHandlers.COMPATIBLE_HEADER));
-            request.params().put(CompatibleHandlers.COMPATIBLE_PARAMS_KEY, compatibleVersion);
+    private void addCompatibleParameter() {
+        if (isRequestCompatible()) {
+            String compatibleVersion = XContentType.parseVersion(header(CompatibleHandlers.COMPATIBLE_HEADER));
+            params().put(CompatibleHandlers.COMPATIBLE_PARAMS_KEY, compatibleVersion);
             //use it so it won't fail request validation with unused parameter
-            request.param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY);
+            param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY);
         }
     }
 
-    public static boolean isRequestCompatible(RestRequest request) {
-        return isHeaderCompatible(request.header(CompatibleHandlers.COMPATIBLE_HEADER));
+    private boolean isRequestCompatible() {
+        return isHeaderCompatible(header(CompatibleHandlers.COMPATIBLE_HEADER));
     }
-    public static boolean isHeaderCompatible(String headerValue) {
+
+    private boolean isHeaderCompatible(String headerValue) {
         String version = XContentType.parseVersion(headerValue);
         return CompatibleHandlers.COMPATIBLE_VERSION.equals(version);
     }
diff --git a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
index 2bdc92ec3fd8f..33c82df8f28a2 100644
--- a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -120,7 +120,7 @@ public void testTypeWhenCompatible() throws IOException {
                 // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
             };
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.compatibleVersion((byte)7);
+            builder.setCompatibleMajorVersion((byte)7);
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
 
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
@@ -129,7 +129,7 @@ public void testTypeWhenCompatible() throws IOException {
         }
 
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.compatibleVersion((byte)6);
+            builder.setCompatibleMajorVersion((byte)6);
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
 
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index dfcfd644dde34..37bf449f44996 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -623,6 +623,39 @@ public void testDispatchRestrictSystemIndices() {
         assertFalse(context.isSystemIndexAccessAllowed());
     }
 
+    public void testDispatchCompatibleHandler() {
+        final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with=7");
+        String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
+        final List<String> contentTypeHeader = Collections.singletonList(mimeType);
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), RestRequest.parseContentType(contentTypeHeader)).withPath("/foo")
+            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
+            .build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                XContentBuilder xContentBuilder = channel.newBuilder();
+                assertThat(xContentBuilder.getCompatibleMajorVersion(), equalTo((byte) 7));
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+
+            @Override
+            public boolean compatibilityRequired() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
     private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
         HttpServerTransport {
 

From f2db19f600d4e9f8572253cd7a155ad9aee7d872 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Tue, 10 Mar 2020 19:45:53 -0500
Subject: [PATCH 08/51] introduce a module to house the REST code

---
 build.gradle                                  |  1 +
 modules/rest-compatibility/build.gradle       | 25 +++++++++
 .../rest/compat/RestCompatPlugin.java         | 33 ++++++++++++
 .../rest/compat/version7/RestGetActionV7.java | 50 +++++++++++++++++
 .../elasticsearch/action/ActionModule.java    |  1 -
 .../rest/action/document/RestGetAction.java   | 53 +++----------------
 6 files changed, 117 insertions(+), 46 deletions(-)
 create mode 100644 modules/rest-compatibility/build.gradle
 create mode 100644 modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
 create mode 100644 modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java

diff --git a/build.gradle b/build.gradle
index 8176f88dab74c..5cee1f108cd69 100644
--- a/build.gradle
+++ b/build.gradle
@@ -114,6 +114,7 @@ subprojects {
       ':distribution:tools:keystore-cli',
       ':distribution:tools:launchers',
       ':distribution:tools:plugin-cli',
+      ':modules:rest-compatibility',
       ':qa:os',
       ':qa:wildfly',
       ':x-pack:plugin:autoscaling',
diff --git a/modules/rest-compatibility/build.gradle b/modules/rest-compatibility/build.gradle
new file mode 100644
index 0000000000000..716c0fbfc0887
--- /dev/null
+++ b/modules/rest-compatibility/build.gradle
@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+  description 'Adds a compatiblity layer for the prior major versions REST API'
+  classname 'org.elasticsearch.rest.compat.RestCompatPlugin'
+}
+
+integTest.enabled = false
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
new file mode 100644
index 0000000000000..a0f08e86195b9
--- /dev/null
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
@@ -0,0 +1,33 @@
+package org.elasticsearch.rest.compat;
+
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.rest.compat.version7.RestGetActionV7;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+public class RestCompatPlugin extends Plugin implements ActionPlugin {
+
+    @Override
+    public List<RestHandler> getRestHandlers(
+        Settings settings,
+        RestController restController,
+        ClusterSettings clusterSettings,
+        IndexScopedSettings indexScopedSettings,
+        SettingsFilter settingsFilter,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<DiscoveryNodes> nodesInCluster
+    ) {
+
+        return List.of(new RestGetActionV7());
+    }
+}
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
new file mode 100644
index 0000000000000..79548f7dedbb6
--- /dev/null
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -0,0 +1,50 @@
+package org.elasticsearch.rest.compat.version7;
+
+import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.document.RestGetAction;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Consumer;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
+
+public class RestGetActionV7 extends RestGetAction {
+
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
+    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
+        + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
+    private static final Consumer<RestRequest> DEPRECATION_WARNING = r -> deprecationLogger.deprecatedAndMaybeLog(
+        "get_with_types",
+        TYPES_DEPRECATION_MESSAGE
+    );
+
+    @Override
+    public List<Route> routes() {
+        assert Version.CURRENT.major == 8 : "REST API compatibility for version 7 is only supported on version 8";
+
+        return unmodifiableList(asList(new Route(GET, "/{index}/{type}/{id}"), new Route(HEAD, "/{index}/{type}/{id}")));
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+        DEPRECATION_WARNING.accept(request);
+        CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+        return super.prepareRequest(request, client);
+    }
+
+
+
+    @Override
+    public boolean compatibilityRequired() {
+        return true;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 0c28c51528ab3..bf3abae1ef2de 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -641,7 +641,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestIndexAction.CompatibleCreateHandler());
         registerHandler.accept(new RestIndexAction.CompatibleAutoIdHandler(nodesInCluster));
         registerHandler.accept(new RestGetAction());
-        registerHandler.accept(new RestGetAction.CompatibleRestGetAction());
         registerHandler.accept(new RestGetSourceAction());
         registerHandler.accept(new RestMultiGetAction(settings));
         registerHandler.accept(new RestDeleteAction());
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index a5b132c2556d8..2996af352ddef 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -19,17 +19,12 @@
 
 package org.elasticsearch.rest.action.document;
 
-import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.CompatibleHandlers;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestActions;
@@ -38,11 +33,7 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.List;
-import java.util.function.Consumer;
 
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
@@ -50,12 +41,6 @@
 
 public class RestGetAction extends BaseRestHandler {
 
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
-    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " +
-        "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
-    private static final Consumer<RestRequest> DEPRECATION_WARNING = r -> deprecationLogger.deprecatedAndMaybeLog("get_with_types",TYPES_DEPRECATION_MESSAGE);
-
-
     @Override
     public String getName() {
         return "document_get_action";
@@ -92,7 +77,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         getRequest.versionType(VersionType.fromString(request.param("version_type"), getRequest.versionType()));
 
         getRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request));
-
         return channel -> client.get(getRequest, new RestToXContentListener<GetResponse>(channel) {
             @Override
             protected RestStatus getStatus(final GetResponse response) {
@@ -101,33 +85,12 @@ protected RestStatus getStatus(final GetResponse response) {
         });
     }
 
-    public static class CompatibleRestGetAction extends RestGetAction {
-
-        private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
-        private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " +
-            "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
-        private static final Consumer<RestRequest> DEPRECATION_WARNING = r -> deprecationLogger.deprecatedAndMaybeLog("get_with_types",TYPES_DEPRECATION_MESSAGE);
-
-
-        @Override
-        public List<Route> routes() {
-            assert Version.CURRENT.major == 8 : "REST API compatilbity for version 7 is only supported on version 8";
-
-            return unmodifiableList(asList(
-                new Route(GET, "/{index}/{type}/{id}"),
-                new Route(HEAD, "/{index}/{type}/{id}")));
-        }
-
-        @Override
-        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-            DEPRECATION_WARNING.accept(request);
-            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
-            return super.prepareRequest(request, client);
-        }
-
-        @Override
-        public boolean compatibilityRequired() {
-            return true;
-        }
-    }
+//    protected RestChannelConsumer response(final NodeClient client, GetRequest getRequest){
+//        return channel -> client.get(getRequest, new RestToXContentListener<GetResponse>(channel) {
+//            @Override
+//            protected RestStatus getStatus(final GetResponse response) {
+//                return response.isExists() ? OK : NOT_FOUND;
+//            }
+//        });
+//    }
 }

From b83b4ce8f6da0e244dc670406858d6c98c06e65d Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Tue, 10 Mar 2020 21:03:32 -0500
Subject: [PATCH 09/51] fix preCommit

---
 modules/rest-compatibility/build.gradle       |  1 +
 .../rest/compat/RestCompatPlugin.java         | 19 +++++++++++++++++
 .../rest/compat/version7/RestGetActionV7.java | 21 +++++++++++++++++--
 .../rest/CompatibleHandlers.java              | 19 +++++++++++++++++
 .../elasticsearch/rest/RestController.java    |  3 ++-
 .../rest/action/document/RestGetAction.java   |  9 --------
 .../rest/action/document/RestIndexAction.java |  2 --
 7 files changed, 60 insertions(+), 14 deletions(-)

diff --git a/modules/rest-compatibility/build.gradle b/modules/rest-compatibility/build.gradle
index 716c0fbfc0887..7e742cc35910a 100644
--- a/modules/rest-compatibility/build.gradle
+++ b/modules/rest-compatibility/build.gradle
@@ -23,3 +23,4 @@ esplugin {
 }
 
 integTest.enabled = false
+test.enabled = false
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
index a0f08e86195b9..4e45136cc0caa 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package org.elasticsearch.rest.compat;
 
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index 79548f7dedbb6..ce295614cf891 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package org.elasticsearch.rest.compat.version7;
 
 import org.apache.logging.log4j.LogManager;
@@ -41,8 +60,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient
         return super.prepareRequest(request, client);
     }
 
-
-
     @Override
     public boolean compatibilityRequired() {
         return true;
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
index 09020e794ed8a..dde2891366c57 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package org.elasticsearch.rest;
 
 import org.elasticsearch.common.logging.DeprecationLogger;
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 06f3588138d03..9e12d6a0e45f0 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -329,7 +329,8 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
                   }
                 } else {
                     if(handler.compatibilityRequired() == false //regular (not removed) handlers are always dispatched
-                        || CompatibleHandlers.isV7Compatible(request)) { //handlers that were registered compatible, require request to be compatible
+                        //handlers that were registered compatible, require request to be compatible
+                        || CompatibleHandlers.isV7Compatible(request)) {
                         dispatchRequest(request, channel, handler);
                     } else {
                         handleCompatibleNotAllowed(rawPath, channel);
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index 2996af352ddef..138e8a0b35973 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -84,13 +84,4 @@ protected RestStatus getStatus(final GetResponse response) {
             }
         });
     }
-
-//    protected RestChannelConsumer response(final NodeClient client, GetRequest getRequest){
-//        return channel -> client.get(getRequest, new RestToXContentListener<GetResponse>(channel) {
-//            @Override
-//            protected RestStatus getStatus(final GetResponse response) {
-//                return response.isExists() ? OK : NOT_FOUND;
-//            }
-//        });
-//    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index 7f5c105cd71a0..041babefba4be 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -25,12 +25,10 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.CompatibleHandlers;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;

From 5c4a02dc991b3de482eb271b0da1cbce5b397071 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 11 Mar 2020 10:37:04 +0100
Subject: [PATCH 10/51] move restindex compatible handlers to
 rest-compatibility module. 228 failing

---
 .../rest/compat/RestCompatPlugin.java         |  9 +-
 .../rest/compat/version7/RestGetActionV7.java |  4 +-
 .../compat/version7/RestIndexActionV7.java    | 99 +++++++++++++++++++
 .../elasticsearch/action/ActionModule.java    |  3 -
 .../rest/action/document/RestIndexAction.java | 67 -------------
 5 files changed, 109 insertions(+), 73 deletions(-)
 create mode 100644 modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
index 4e45136cc0caa..aa72620fbf3c2 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.compat.version7.RestGetActionV7;
+import org.elasticsearch.rest.compat.version7.RestIndexActionV7;
 
 import java.util.List;
 import java.util.function.Supplier;
@@ -46,7 +47,11 @@ public List<RestHandler> getRestHandlers(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<DiscoveryNodes> nodesInCluster
     ) {
-
-        return List.of(new RestGetActionV7());
+        return List.of(
+            new RestGetActionV7(),
+            new RestIndexActionV7.CompatibleRestIndexAction(),
+            new RestIndexActionV7.CompatibleCreateHandler(),
+            new RestIndexActionV7.CompatibleAutoIdHandler(nodesInCluster)
+        );
     }
 }
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index ce295614cf891..3035725f57677 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -50,7 +50,9 @@ public class RestGetActionV7 extends RestGetAction {
     public List<Route> routes() {
         assert Version.CURRENT.major == 8 : "REST API compatibility for version 7 is only supported on version 8";
 
-        return unmodifiableList(asList(new Route(GET, "/{index}/{type}/{id}"), new Route(HEAD, "/{index}/{type}/{id}")));
+        return List.of(
+            new Route(GET, "/{index}/{type}/{id}"),
+            new Route(HEAD, "/{index}/{type}/{id}"));
     }
 
     @Override
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
new file mode 100644
index 0000000000000..dd2f556773c92
--- /dev/null
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -0,0 +1,99 @@
+package org.elasticsearch.rest.compat.version7;
+
+import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.document.RestIndexAction;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+public class RestIndexActionV7 {
+    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " +
+        "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " +
+        "or /{index}/_create/{id}).";
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
+        LogManager.getLogger(RestIndexAction.class));
+    private static final Consumer<RestRequest> DEPRECATION_WARNING =
+        r -> deprecationLogger.deprecatedAndMaybeLog("index_with_types",TYPES_DEPRECATION_MESSAGE);
+
+    public static class CompatibleRestIndexAction extends RestIndexAction {
+        @Override
+        public List<Route> routes() {
+            assert Version.CURRENT.major == 8 : "REST API compatilbity for version 7 is only supported on version 8";
+
+            return List.of(
+                new Route(POST, "/{index}/{type}/{id}"),
+                new Route(PUT, "/{index}/{type}/{id}"));
+        }
+
+        @Override
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+            DEPRECATION_WARNING.accept(request);
+            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            return super.prepareRequest(request, client);
+        }
+
+        @Override
+        public boolean compatibilityRequired() {
+            return true;
+        }
+    }
+
+    public static class CompatibleCreateHandler extends RestIndexAction.CreateHandler {
+        @Override
+        public List<Route> routes() {
+            return unmodifiableList(asList(
+                new Route(POST, "/{index}/{type}/{id}/_create"),
+                new Route(PUT, "/{index}/{type}/{id}/_create")));
+        }
+
+        @Override
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+            DEPRECATION_WARNING.accept(request);
+            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            return super.prepareRequest(request, client);
+        }
+
+        @Override
+        public boolean compatibilityRequired() {
+            return true;
+        }
+    }
+
+    public static final class CompatibleAutoIdHandler extends RestIndexAction.AutoIdHandler {
+
+        public CompatibleAutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
+            super(nodesInCluster);
+        }
+
+        @Override
+        public List<Route> routes() {
+            return singletonList(new Route(POST, "/{index}/{type}"));
+        }
+
+        @Override
+        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
+            DEPRECATION_WARNING.accept(request);
+            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            return super.prepareRequest(request, client);
+        }
+
+        @Override
+        public boolean compatibilityRequired() {
+            return true;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index bf3abae1ef2de..affbb7a41dd31 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -637,9 +637,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestIndexAction());
         registerHandler.accept(new CreateHandler());
         registerHandler.accept(new AutoIdHandler(nodesInCluster));
-        registerHandler.accept(new RestIndexAction.CompatibleRestIndexAction());
-        registerHandler.accept(new RestIndexAction.CompatibleCreateHandler());
-        registerHandler.accept(new RestIndexAction.CompatibleAutoIdHandler(nodesInCluster));
         registerHandler.accept(new RestGetAction());
         registerHandler.accept(new RestGetSourceAction());
         registerHandler.accept(new RestMultiGetAction(settings));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index 041babefba4be..20cda9f3ceff0 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -66,29 +66,6 @@ public String getName() {
         return "document_index_action";
     }
 
-    public static class CompatibleRestIndexAction extends  RestIndexAction{
-        @Override
-        public List<Route> routes() {
-            assert Version.CURRENT.major == 8 : "REST API compatilbity for version 7 is only supported on version 8";
-
-            return List.of(
-                new Route(POST, "/{index}/{type}/{id}"),
-                new Route(PUT, "/{index}/{type}/{id}"));
-        }
-
-        @Override
-        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-            DEPRECATION_WARNING.accept(request);
-            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
-            return super.prepareRequest(request, client);
-        }
-
-        @Override
-        public boolean compatibilityRequired() {
-            return true;
-        }
-    }
-
     public static class CreateHandler extends RestIndexAction {
 
         @Override
@@ -117,27 +94,6 @@ void validateOpType(String opType) {
         }
     }
 
-    public static class CompatibleCreateHandler extends CreateHandler {
-        @Override
-        public List<Route> routes() {
-            return unmodifiableList(asList(
-                new Route(POST, "/{index}/{type}/{id}/_create"),
-                new Route(PUT, "/{index}/{type}/{id}/_create")));
-        }
-
-        @Override
-        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-            DEPRECATION_WARNING.accept(request);
-            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
-            return super.prepareRequest(request, client);
-        }
-
-        @Override
-        public boolean compatibilityRequired() {
-            return true;
-        }
-    }
-
     public static class AutoIdHandler extends RestIndexAction {
 
         private final Supplier<DiscoveryNodes> nodesInCluster;
@@ -166,29 +122,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient
             return super.prepareRequest(request, client);
         }
     }
-    public static final class CompatibleAutoIdHandler extends AutoIdHandler {
-
-        public CompatibleAutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
-            super(nodesInCluster);
-        }
-
-        @Override
-        public List<Route> routes() {
-            return singletonList(new Route(POST, "/{index}/{type}"));
-        }
-
-        @Override
-        public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-            DEPRECATION_WARNING.accept(request);
-            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
-            return super.prepareRequest(request, client);
-        }
-
-        @Override
-        public boolean compatibilityRequired() {
-            return true;
-        }
-    }
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {

From 0ef7e1840c9eb391299cfbfedf760f88cde65c0f Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 11 Mar 2020 19:36:41 +0100
Subject: [PATCH 11/51] moving test classes and compat related code to separate
 v7 module

code review follow up
---
 .../common/xcontent/XContentType.java         | 12 ++--
 .../rest/compat/RestCompatPlugin.java         | 17 +++--
 .../compat/version7}/CompatibleHandlers.java  | 16 +----
 .../rest/compat/version7/RestGetActionV7.java |  4 +-
 .../compat/version7/RestIndexActionV7.java    |  2 +-
 .../action/DocWriteResponseTests.java         | 68 +++++++++++++++++++
 .../rest/compat/AbstractCompatRestTest.java   |  6 +-
 .../rest/AbstractRestChannel.java             |  2 +-
 .../rest/CompatibleConstants.java             | 37 ++++++++++
 .../elasticsearch/rest/RestController.java    | 10 ++-
 .../org/elasticsearch/rest/RestRequest.java   | 10 +--
 .../rest/action/document/RestIndexAction.java | 15 ----
 .../action/DocWriteResponseTests.java         | 28 --------
 .../common/xcontent/XContentTypeTests.java    | 40 +++++++++--
 .../http/DefaultRestChannelTests.java         | 11 +--
 .../rest/RestControllerTests.java             |  3 +-
 16 files changed, 188 insertions(+), 93 deletions(-)
 rename {server/src/main/java/org/elasticsearch/rest => modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7}/CompatibleHandlers.java (69%)
 create mode 100644 modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index 89f9c46f38ce4..43d36a800d345 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -116,6 +116,10 @@ public XContent xContent() {
         }
     };
 
+    private static final Pattern COMPATIBLE_API_HEADER_PATTERN = Pattern.compile(
+        "application/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?",
+        Pattern.CASE_INSENSITIVE);
+
     /**
      * Accepts either a format string, which is equivalent to {@link XContentType#shortName()} or a media type that optionally has
      * parameters and attempts to match the value to an {@link XContentType}. The comparisons are done in lower case format and this method
@@ -161,13 +165,11 @@ public static XContentType fromMediaType(String mediaTypeHeaderValue) {
         return null;
     }
 
-    static Pattern pattern = Pattern.compile("application/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?");
-
     public static String parseMediaType(String mediaType) {
         if (mediaType != null) {
-            Matcher matcher = pattern.matcher(mediaType);
+            Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
             if (matcher.find()) {
-                return "application/"+matcher.group(2);
+                return "application/" + matcher.group(2).toLowerCase();
             }
         }
 
@@ -176,7 +178,7 @@ public static String parseMediaType(String mediaType) {
 
     public static String parseVersion(String mediaType){
         if(mediaType != null){
-            Matcher matcher = pattern.matcher(mediaType);
+            Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
             if (matcher.find() && "vnd.elasticsearch+".equals(matcher.group(1))) {
 
                 return matcher.group(4);
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
index aa72620fbf3c2..18b803f41ab4d 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/RestCompatPlugin.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest.compat;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -32,6 +33,7 @@
 import org.elasticsearch.rest.compat.version7.RestGetActionV7;
 import org.elasticsearch.rest.compat.version7.RestIndexActionV7;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -47,11 +49,14 @@ public List<RestHandler> getRestHandlers(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<DiscoveryNodes> nodesInCluster
     ) {
-        return List.of(
-            new RestGetActionV7(),
-            new RestIndexActionV7.CompatibleRestIndexAction(),
-            new RestIndexActionV7.CompatibleCreateHandler(),
-            new RestIndexActionV7.CompatibleAutoIdHandler(nodesInCluster)
-        );
+        if (Version.CURRENT.major == 8) {
+            return List.of(
+                new RestGetActionV7(),
+                new RestIndexActionV7.CompatibleRestIndexAction(),
+                new RestIndexActionV7.CompatibleCreateHandler(),
+                new RestIndexActionV7.CompatibleAutoIdHandler(nodesInCluster)
+            );
+        }
+        return Collections.emptyList();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/CompatibleHandlers.java
similarity index 69%
rename from server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
rename to modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/CompatibleHandlers.java
index dde2891366c57..cbd61cd0ea9df 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/CompatibleHandlers.java
@@ -17,22 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.rest;
+package org.elasticsearch.rest.compat.version7;
 
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.rest.RestRequest;
 
 import java.util.function.Consumer;
 
 public class CompatibleHandlers {
 
-    /**
-     * TODO revisit when https://github.com/elastic/elasticsearch/issues/52370 is resolved
-     */
-    public static final String COMPATIBLE_HEADER = "Accept";
-    public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
-    public static final String COMPATIBLE_VERSION = "7";
-
     public static Consumer<RestRequest> consumeParameterType(DeprecationLogger deprecationLogger) {
         String TYPES_DEPRECATION_MESSAGE = "[types removal] Using type as a path parameter is deprecated.";
 
@@ -42,9 +35,4 @@ public static Consumer<RestRequest> consumeParameterType(DeprecationLogger depre
         };
     }
 
-    public static boolean isV7Compatible(ToXContent.Params params) {
-        String param = params.param(COMPATIBLE_PARAMS_KEY);
-        return COMPATIBLE_VERSION.equals(param);
-    }
-
 }
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index 3035725f57677..fb0d91dc92c96 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.CompatibleConstants;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.document.RestGetAction;
 
@@ -31,8 +31,6 @@
 import java.util.List;
 import java.util.function.Consumer;
 
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
index dd2f556773c92..2787407478906 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -5,7 +5,7 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.CompatibleConstants;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.document.RestIndexAction;
 
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
new file mode 100644
index 0000000000000..09acb4a2d0047
--- /dev/null
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+
+public class DocWriteResponseTests extends ESTestCase {
+
+    public void testTypeWhenCompatible() throws IOException {
+        DocWriteResponse response =
+            new DocWriteResponse(
+                new ShardId("index", "uuid", 0),
+                "id",
+                SequenceNumbers.UNASSIGNED_SEQ_NO,
+                17,
+                0,
+                DocWriteResponse.Result.CREATED) {
+                // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
+            };
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.setCompatibleMajorVersion((byte)7);
+            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+                assertThat(parser.map(), hasEntry(DocWriteResponse.TYPE_FIELD_NAME,DocWriteResponse.SINGLE_MAPPING_TYPE.toString()));
+            }
+        }
+
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.setCompatibleMajorVersion((byte)6);
+            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+                assertThat(parser.map(), not(hasKey(DocWriteResponse.TYPE_FIELD_NAME)));
+            }
+        }
+    }
+}
diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index ef5de432bef71..e8295d5462470 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -22,7 +22,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.CompatibleConstants;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.elasticsearch.test.rest.yaml.section.DoSection;
@@ -80,12 +80,12 @@ private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
 
             String compatibleHeader = createCompatibleHeader();
             doSection.getApiCallSection()
-                     .addHeaders(Collections.singletonMap(CompatibleHandlers.COMPATIBLE_HEADER, compatibleHeader));
+                     .addHeaders(Collections.singletonMap(CompatibleConstants.COMPATIBLE_HEADER, compatibleHeader));
         });
     }
 
     private static String createCompatibleHeader() {
-        return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleHandlers.COMPATIBLE_VERSION;
+        return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleConstants.COMPATIBLE_VERSION;
     }
 
 
diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
index 5beaa6fd8eaf2..dc378ab0d4117 100644
--- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
+++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
@@ -127,7 +127,7 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu
         }
 
         builder.humanReadable(human);
-        String compatibleVersion = request.param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY);
+        String compatibleVersion = request.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
         builder.setCompatibleMajorVersion(compatibleVersion == null ? -1 : Byte.parseByte(compatibleVersion));
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
new file mode 100644
index 0000000000000..a86438695039f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.xcontent.ToXContent;
+
+import java.util.function.Consumer;
+
+public class CompatibleConstants {
+
+    /**
+     * TODO revisit when https://github.com/elastic/elasticsearch/issues/52370 is resolved
+     */
+    public static final String COMPATIBLE_HEADER = "Accept";
+    public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
+    public static final String COMPATIBLE_VERSION = "" + (Version.CURRENT.major - 1);
+
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 9e12d6a0e45f0..4b7a25839749e 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.core.internal.io.Streams;
@@ -55,6 +56,8 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.rest.BytesRestResponse.TEXT_CONTENT_TYPE;
+import static org.elasticsearch.rest.CompatibleConstants.COMPATIBLE_PARAMS_KEY;
+import static org.elasticsearch.rest.CompatibleConstants.COMPATIBLE_VERSION;
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
 import static org.elasticsearch.rest.RestStatus.METHOD_NOT_ALLOWED;
@@ -330,7 +333,7 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
                 } else {
                     if(handler.compatibilityRequired() == false //regular (not removed) handlers are always dispatched
                         //handlers that were registered compatible, require request to be compatible
-                        || CompatibleHandlers.isV7Compatible(request)) {
+                        || isV7Compatible(request)) {
                         dispatchRequest(request, channel, handler);
                     } else {
                         handleCompatibleNotAllowed(rawPath, channel);
@@ -346,6 +349,11 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
         handleBadRequest(uri, requestMethod, channel);
     }
 
+    public static boolean isV7Compatible(ToXContent.Params params) {
+        String param = params.param(COMPATIBLE_PARAMS_KEY);
+        return COMPATIBLE_VERSION.equals(param);
+    }
+
     private void handleCompatibleNotAllowed(String rawPath, RestChannel channel) throws IOException {
         String msg = "Compatible api can be only accessed with Compatible Header. Path used: " + rawPath;
         BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_FOUND, msg);
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index b54f67c42f589..63ebe4bf6e185 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -139,20 +139,20 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
 
     private void addCompatibleParameter() {
         if (isRequestCompatible()) {
-            String compatibleVersion = XContentType.parseVersion(header(CompatibleHandlers.COMPATIBLE_HEADER));
-            params().put(CompatibleHandlers.COMPATIBLE_PARAMS_KEY, compatibleVersion);
+            String compatibleVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_HEADER));
+            params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, compatibleVersion);
             //use it so it won't fail request validation with unused parameter
-            param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY);
+            param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
         }
     }
 
     private boolean isRequestCompatible() {
-        return isHeaderCompatible(header(CompatibleHandlers.COMPATIBLE_HEADER));
+        return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_HEADER));
     }
 
     private boolean isHeaderCompatible(String headerValue) {
         String version = XContentType.parseVersion(headerValue);
-        return CompatibleHandlers.COMPATIBLE_VERSION.equals(version);
+        return CompatibleConstants.COMPATIBLE_VERSION.equals(version);
     }
 
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index 20cda9f3ceff0..326d3d46f29c4 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -19,16 +19,13 @@
 
 package org.elasticsearch.rest.action.document;
 
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.CompatibleHandlers;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
@@ -36,24 +33,12 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
-import java.util.function.Consumer;
 import java.util.function.Supplier;
 
-import static java.util.Arrays.asList;
-import static java.util.Collections.singletonList;
-import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestIndexAction extends BaseRestHandler {
-    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " +
-        "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " +
-        "or /{index}/_create/{id}).";
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
-        LogManager.getLogger(RestIndexAction.class));
-    private static final Consumer<RestRequest> DEPRECATION_WARNING =
-        r -> deprecationLogger.deprecatedAndMaybeLog("index_with_types",TYPES_DEPRECATION_MESSAGE);
-
     @Override
     public List<Route> routes() {
         return List.of(
diff --git a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
index 33c82df8f28a2..7751f002d9b05 100644
--- a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -108,33 +108,5 @@ public void testToXContentDoesntIncludeForcedRefreshUnlessForced() throws IOExce
         }
     }
 
-    public void testTypeWhenCompatible() throws IOException {
-        DocWriteResponse response =
-            new DocWriteResponse(
-                new ShardId("index", "uuid", 0),
-                "id",
-                SequenceNumbers.UNASSIGNED_SEQ_NO,
-                17,
-                0,
-                Result.CREATED) {
-                // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
-            };
-        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.setCompatibleMajorVersion((byte)7);
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
 
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                assertThat(parser.map(), hasEntry(DocWriteResponse.TYPE_FIELD_NAME,DocWriteResponse.SINGLE_MAPPING_TYPE.toString()));
-            }
-        }
-
-        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.setCompatibleMajorVersion((byte)6);
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                assertThat(parser.map(), not(hasKey(DocWriteResponse.TYPE_FIELD_NAME)));
-            }
-        }
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
index a2b89b2bf542f..c79beedc7ca18 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
@@ -85,10 +85,40 @@ public void testFromRubbish() throws Exception {
         assertThat(XContentType.fromMediaTypeOrFormat("gobbly;goop"), nullValue());
     }
 
-    public void testMediaType() throws Exception {
-        String mediaType = XContentType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=7");
-        assertThat(mediaType,equalTo("application/json"));
-        mediaType = XContentType.parseMediaType("application/json");
-        assertThat(mediaType,equalTo("application/json"));
+    public void testMediaType() {
+        byte version = randomByte();
+        assertThat(XContentType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=" + version),
+            equalTo("application/json"));
+        assertThat(XContentType.parseMediaType("application/vnd.elasticsearch+cbor;compatible-with=" + version),
+            equalTo("application/cbor"));
+        assertThat(XContentType.parseMediaType("application/vnd.elasticsearch+smile;compatible-with=" + version),
+            equalTo("application/smile"));
+        assertThat(XContentType.parseMediaType("application/json"),
+            equalTo("application/json"));
+
+
+        assertThat(XContentType.parseMediaType("APPLICATION/VND.ELASTICSEARCH+JSON;COMPATIBLE-WITH=" + version),
+            equalTo("application/json"));
+        assertThat(XContentType.parseMediaType("APPLICATION/JSON"),
+            equalTo("application/json"));
+    }
+
+
+    public void testVersionParsing() {
+        byte version = randomByte();
+        assertThat(XContentType.parseVersion("application/vnd.elasticsearch+json;compatible-with=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("application/vnd.elasticsearch+cbor;compatible-with=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("application/vnd.elasticsearch+smile;compatible-with=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("application/json"),
+            nullValue());
+
+
+        assertThat(XContentType.parseVersion("APPLICATION/VND.ELASTICSEARCH+JSON;COMPATIBLE-WITH=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("APPLICATION/JSON"),
+            nullValue());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
index b1f43f010efcf..8c5beabfe0016 100644
--- a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
+++ b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
@@ -37,7 +37,7 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.CompatibleHandlers;
+import org.elasticsearch.rest.CompatibleConstants;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -214,12 +214,13 @@ public void testHeadersSet() {
         assertEquals(resp.contentType(), headers.get(DefaultRestChannel.CONTENT_TYPE).get(0));
     }
 
-    public void testCompatibleParamIsSet(){
+    public void testCompatibleParamIsSet() {
+        final byte version = randomByte();
         final TestRequest httpRequest = new TestRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/");
-        httpRequest.getHeaders().put(HttpHeaders.ACCEPT, List.of("application/vnd.elasticsearch+json;compatible-with=7"));
+        httpRequest.getHeaders().put(HttpHeaders.ACCEPT, List.of("application/vnd.elasticsearch+json;compatible-with=" + version));
         final RestRequest request = RestRequest.request(xContentRegistry(), httpRequest, httpChannel);
 
-        assertEquals("7", request.param(CompatibleHandlers.COMPATIBLE_PARAMS_KEY));
+        assertEquals("" + version, request.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY));
     }
 
     public void testCookiesSet() {
@@ -414,7 +415,7 @@ private TestResponse executeRequest(final Settings settings, final String origin
         return responseCaptor.getValue();
     }
 
-    private static class TestRequest implements HttpRequest {
+    public static class TestRequest implements HttpRequest {
 
         private final Supplier<HttpVersion> version;
         private final RestRequest.Method method;
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 37bf449f44996..b8d4a63977bbf 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -632,11 +632,12 @@ public void testDispatchCompatibleHandler() {
             .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
             .build();
         AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        final byte version = randomByte();
         restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
             @Override
             public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
                 XContentBuilder xContentBuilder = channel.newBuilder();
-                assertThat(xContentBuilder.getCompatibleMajorVersion(), equalTo((byte) 7));
+                assertThat(xContentBuilder.getCompatibleMajorVersion(), equalTo(version));
                 channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
             }
 

From 835ce563ffe8ad47c13cb19d1679f3bf208995dd Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 13 Mar 2020 11:27:25 +0100
Subject: [PATCH 12/51] test class rename and return 400 when compatible header
 not present

---
 .../rest/compat/version7/RestGetActionV7.java            | 1 -
 .../rest/compat/version7/RestIndexActionV7.java          | 1 -
 ...teResponseTests.java => DocWriteResponseV7Tests.java} | 2 +-
 .../rest/compat/AbstractCompatRestTest.java              | 8 ++++++--
 .../main/java/org/elasticsearch/rest/RestController.java | 9 +--------
 5 files changed, 8 insertions(+), 13 deletions(-)
 rename modules/rest-compatibility/src/test/java/org/elasticsearch/action/{DocWriteResponseTests.java => DocWriteResponseV7Tests.java} (97%)

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index fb0d91dc92c96..4e516c06bf184 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.rest.CompatibleConstants;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.document.RestGetAction;
 
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
index 2787407478906..5f7e931624ec3 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -5,7 +5,6 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.rest.CompatibleConstants;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.document.RestIndexAction;
 
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
similarity index 97%
rename from modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
rename to modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
index 09acb4a2d0047..299e5697424df 100644
--- a/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
@@ -34,7 +34,7 @@
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.not;
 
-public class DocWriteResponseTests extends ESTestCase {
+public class DocWriteResponseV7Tests extends ESTestCase {
 
     public void testTypeWhenCompatible() throws IOException {
         DocWriteResponse response =
diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index e8295d5462470..94b28abc5017e 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -28,9 +28,9 @@
 import org.elasticsearch.test.rest.yaml.section.DoSection;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 
+import java.net.http.HttpHeaders;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -79,8 +79,12 @@ private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
             doSection.setIgnoreWarnings(true);
 
             String compatibleHeader = createCompatibleHeader();
+            //TODO decide which one to use - Accept or Content-Type
             doSection.getApiCallSection()
-                     .addHeaders(Collections.singletonMap(CompatibleConstants.COMPATIBLE_HEADER, compatibleHeader));
+                     .addHeaders(Map.of(
+                         CompatibleConstants.COMPATIBLE_HEADER, compatibleHeader,
+                         "Content-Type", compatibleHeader
+                         ));
         });
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 4b7a25839749e..06b3b9f61f9ae 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -336,7 +336,7 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
                         || isV7Compatible(request)) {
                         dispatchRequest(request, channel, handler);
                     } else {
-                        handleCompatibleNotAllowed(rawPath, channel);
+                        handleBadRequest(uri, requestMethod, channel);
                     }
                     return;
                 }
@@ -354,13 +354,6 @@ public static boolean isV7Compatible(ToXContent.Params params) {
         return COMPATIBLE_VERSION.equals(param);
     }
 
-    private void handleCompatibleNotAllowed(String rawPath, RestChannel channel) throws IOException {
-        String msg = "Compatible api can be only accessed with Compatible Header. Path used: " + rawPath;
-        BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_FOUND, msg);
-
-        channel.sendResponse(bytesRestResponse);
-    }
-
     Iterator<MethodHandlers> getAllHandlers(@Nullable Map<String, String> requestParamsRef, String rawPath) {
         final Supplier<Map<String, String>> paramsSupplier;
         if (requestParamsRef == null) {

From f4402316e351d2738d584cbc128ef09fd55f87ba Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 13 Mar 2020 11:50:33 +0100
Subject: [PATCH 13/51] clean up deprecation warnings and remove use of
 consumers

---
 .../compat/version7/CompatibleHandlers.java   | 38 -------------------
 1 file changed, 38 deletions(-)
 delete mode 100644 modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/CompatibleHandlers.java

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/CompatibleHandlers.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/CompatibleHandlers.java
deleted file mode 100644
index cbd61cd0ea9df..0000000000000
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/CompatibleHandlers.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.compat.version7;
-
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.rest.RestRequest;
-
-import java.util.function.Consumer;
-
-public class CompatibleHandlers {
-
-    public static Consumer<RestRequest> consumeParameterType(DeprecationLogger deprecationLogger) {
-        String TYPES_DEPRECATION_MESSAGE = "[types removal] Using type as a path parameter is deprecated.";
-
-        return r -> {
-            deprecationLogger.deprecatedAndMaybeLog("create_index_with_types", TYPES_DEPRECATION_MESSAGE);
-            r.param("type");
-        };
-    }
-
-}

From 84f1ddeef546172bf0eafdf1c879a371ea8e675f Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 13 Mar 2020 13:19:18 +0100
Subject: [PATCH 14/51] v7 compatible actions warnings tests

---
 .../rest/compat/version7/RestGetActionV7.java | 11 +--
 .../compat/version7/RestIndexActionV7.java    | 22 ++---
 .../action/DocWriteResponseV7Tests.java       |  3 +-
 .../compat/version7/RestGetActionV7Test.java  | 56 +++++++++++++
 .../version7/RestIndexActionV7Test.java       | 81 +++++++++++++++++++
 .../action/DocWriteResponse.java              |  4 +-
 .../action/DocWriteResponseTests.java         |  2 -
 7 files changed, 155 insertions(+), 24 deletions(-)
 create mode 100644 modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Test.java
 create mode 100644 modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Test.java

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index 4e516c06bf184..e00a600d073df 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -28,7 +28,6 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
@@ -36,12 +35,8 @@
 public class RestGetActionV7 extends RestGetAction {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
-    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
+    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
         + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
-    private static final Consumer<RestRequest> DEPRECATION_WARNING = r -> deprecationLogger.deprecatedAndMaybeLog(
-        "get_with_types",
-        TYPES_DEPRECATION_MESSAGE
-    );
 
     @Override
     public List<Route> routes() {
@@ -54,8 +49,8 @@ public List<Route> routes() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-        DEPRECATION_WARNING.accept(request);
-        CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+        deprecationLogger.deprecatedAndMaybeLog("get_with_types", TYPES_DEPRECATION_MESSAGE);
+        request.param("type");
         return super.prepareRequest(request, client);
     }
 
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
index 5f7e931624ec3..f159801a3ce94 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -10,7 +10,6 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.function.Consumer;
 import java.util.function.Supplier;
 
 import static java.util.Arrays.asList;
@@ -20,13 +19,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestIndexActionV7 {
-    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " +
+    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " +
         "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " +
         "or /{index}/_create/{id}).";
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
         LogManager.getLogger(RestIndexAction.class));
-    private static final Consumer<RestRequest> DEPRECATION_WARNING =
-        r -> deprecationLogger.deprecatedAndMaybeLog("index_with_types",TYPES_DEPRECATION_MESSAGE);
+
+    private static void logDeprecationMessage() {
+        deprecationLogger.deprecatedAndMaybeLog("index_with_types", TYPES_DEPRECATION_MESSAGE);
+    }
 
     public static class CompatibleRestIndexAction extends RestIndexAction {
         @Override
@@ -40,11 +41,12 @@ public List<Route> routes() {
 
         @Override
         public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-            DEPRECATION_WARNING.accept(request);
-            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            logDeprecationMessage();
+            request.param("type");
             return super.prepareRequest(request, client);
         }
 
+
         @Override
         public boolean compatibilityRequired() {
             return true;
@@ -61,8 +63,8 @@ public List<Route> routes() {
 
         @Override
         public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-            DEPRECATION_WARNING.accept(request);
-            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            logDeprecationMessage();
+            request.param("type");
             return super.prepareRequest(request, client);
         }
 
@@ -85,8 +87,8 @@ public List<Route> routes() {
 
         @Override
         public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
-            DEPRECATION_WARNING.accept(request);
-            CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request);
+            logDeprecationMessage();
+            request.param("type");
             return super.prepareRequest(request, client);
         }
 
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
index 299e5697424df..0f3fcd7e85289 100644
--- a/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
@@ -52,7 +53,7 @@ public void testTypeWhenCompatible() throws IOException {
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
 
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                assertThat(parser.map(), hasEntry(DocWriteResponse.TYPE_FIELD_NAME,DocWriteResponse.SINGLE_MAPPING_TYPE.toString()));
+                assertThat(parser.map(), hasEntry(DocWriteResponse.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME));
             }
         }
 
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Test.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Test.java
new file mode 100644
index 0000000000000..b2a53a096adad
--- /dev/null
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Test.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.compat.version7;
+
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class RestGetActionV7Test extends RestActionTestCase {
+    final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with=7");
+    final List<String> contentTypeHeader = Collections.singletonList(mimeType);
+
+    @Before
+    public void setUpAction() {
+        controller().registerHandler(new RestGetActionV7());
+    }
+
+    public void testTypeInPathWithGet() {
+        FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
+            .withPath("/some_index/some_type/some_id");
+        dispatchRequest(deprecatedRequest.withMethod(RestRequest.Method.GET).build());
+        assertWarnings(RestGetActionV7.TYPES_DEPRECATION_MESSAGE);
+    }
+
+    public void testTypeInPathWithHead() {
+        FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
+            .withPath("/some_index/some_type/some_id");
+        dispatchRequest(deprecatedRequest.withMethod(RestRequest.Method.HEAD).build());
+        assertWarnings(RestGetActionV7.TYPES_DEPRECATION_MESSAGE);
+    }
+
+}
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Test.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Test.java
new file mode 100644
index 0000000000000..157a784354422
--- /dev/null
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Test.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.compat.version7;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class RestIndexActionV7Test extends RestActionTestCase {
+
+    final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with=7");
+    final List<String> contentTypeHeader = Collections.singletonList(mimeType);
+
+
+    private final AtomicReference<ClusterState> clusterStateSupplier = new AtomicReference<>();
+
+    @Before
+    public void setUpAction() {
+        controller().registerHandler(new RestIndexActionV7.CompatibleRestIndexAction());
+        controller().registerHandler(new RestIndexActionV7.CompatibleCreateHandler());
+        controller().registerHandler(new RestIndexActionV7.CompatibleAutoIdHandler(() -> clusterStateSupplier.get().nodes()));
+    }
+
+    public void testTypeInPath() {
+        // using CompatibleRestIndexAction
+        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withMethod(RestRequest.Method.PUT)
+            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
+            .withPath("/some_index/some_type/some_id")
+            .build();
+        dispatchRequest(deprecatedRequest);
+        assertWarnings(RestIndexActionV7.TYPES_DEPRECATION_MESSAGE);
+    }
+
+    public void testCreateWithTypeInPath() {
+        // using CompatibleCreateHandler
+        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withMethod(RestRequest.Method.PUT)
+            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
+            .withPath("/some_index/some_type/some_id/_create")
+            .build();
+        dispatchRequest(deprecatedRequest);
+        assertWarnings(RestIndexActionV7.TYPES_DEPRECATION_MESSAGE);
+    }
+
+    public void testAutoIdWithType() {
+        // using CompatibleAutoIdHandler
+
+        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withMethod(RestRequest.Method.PUT)
+            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
+            .withPath("/some_index/some_type/")
+            .build();
+        dispatchRequest(deprecatedRequest);
+        assertWarnings(RestIndexActionV7.TYPES_DEPRECATION_MESSAGE);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index 41fb9d16b96a0..00ab13e24452f 100644
--- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -55,7 +54,6 @@
 public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContentObject {
 
     static final String TYPE_FIELD_NAME = "_type";
-    static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME);
 
     private static final String _SHARDS = "_shards";
     private static final String _INDEX = "_index";
@@ -281,7 +279,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major) {
-            builder.field(TYPE_FIELD_NAME, SINGLE_MAPPING_TYPE);
+            builder.field(TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
         }
         innerToXContent(builder, params);
         builder.endObject();
diff --git a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
index 7751f002d9b05..bb1208bc3bba1 100644
--- a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -107,6 +107,4 @@ public void testToXContentDoesntIncludeForcedRefreshUnlessForced() throws IOExce
             }
         }
     }
-
-
 }

From d106d1ba66027268c061875b3aab86f92bb8c690 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 16 Mar 2020 10:59:17 +0100
Subject: [PATCH 15/51] rename tests and enable them

---
 modules/rest-compatibility/build.gradle                      | 4 ++--
 .../{RestGetActionV7Test.java => RestGetActionV7Tests.java}  | 2 +-
 ...estIndexActionV7Test.java => RestIndexActionV7Tests.java} | 5 ++---
 3 files changed, 5 insertions(+), 6 deletions(-)
 rename modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/{RestGetActionV7Test.java => RestGetActionV7Tests.java} (97%)
 rename modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/{RestIndexActionV7Test.java => RestIndexActionV7Tests.java} (96%)

diff --git a/modules/rest-compatibility/build.gradle b/modules/rest-compatibility/build.gradle
index 7e742cc35910a..ca0401d69fff3 100644
--- a/modules/rest-compatibility/build.gradle
+++ b/modules/rest-compatibility/build.gradle
@@ -22,5 +22,5 @@ esplugin {
   classname 'org.elasticsearch.rest.compat.RestCompatPlugin'
 }
 
-integTest.enabled = false
-test.enabled = false
+integTest.enabled = true
+test.enabled = true
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Test.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Tests.java
similarity index 97%
rename from modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Test.java
rename to modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Tests.java
index b2a53a096adad..31bb32d93ac25 100644
--- a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Test.java
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Tests.java
@@ -28,7 +28,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class RestGetActionV7Test extends RestActionTestCase {
+public class RestGetActionV7Tests extends RestActionTestCase {
     final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with=7");
     final List<String> contentTypeHeader = Collections.singletonList(mimeType);
 
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Test.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Tests.java
similarity index 96%
rename from modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Test.java
rename to modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Tests.java
index 157a784354422..d079f8d9600b3 100644
--- a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Test.java
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Tests.java
@@ -30,7 +30,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
-public class RestIndexActionV7Test extends RestActionTestCase {
+public class RestIndexActionV7Tests extends RestActionTestCase {
 
     final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with=7");
     final List<String> contentTypeHeader = Collections.singletonList(mimeType);
@@ -69,9 +69,8 @@ public void testCreateWithTypeInPath() {
 
     public void testAutoIdWithType() {
         // using CompatibleAutoIdHandler
-
         RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
-            .withMethod(RestRequest.Method.PUT)
+            .withMethod(RestRequest.Method.POST)
             .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
             .withPath("/some_index/some_type/")
             .build();

From cf61bbddd2ebbb318958f9a1a1064aca79615bd3 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 16 Mar 2020 11:03:49 +0100
Subject: [PATCH 16/51] rename isV7Compatible method

---
 .../src/main/java/org/elasticsearch/rest/RestController.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 06b3b9f61f9ae..1f97f8556319f 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -333,7 +333,7 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
                 } else {
                     if(handler.compatibilityRequired() == false //regular (not removed) handlers are always dispatched
                         //handlers that were registered compatible, require request to be compatible
-                        || isV7Compatible(request)) {
+                        || isCompatible(request)) {
                         dispatchRequest(request, channel, handler);
                     } else {
                         handleBadRequest(uri, requestMethod, channel);
@@ -349,7 +349,7 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
         handleBadRequest(uri, requestMethod, channel);
     }
 
-    public static boolean isV7Compatible(ToXContent.Params params) {
+    private boolean isCompatible(ToXContent.Params params) {
         String param = params.param(COMPATIBLE_PARAMS_KEY);
         return COMPATIBLE_VERSION.equals(param);
     }

From f00f438b8c66a39938a24fc99b523787229be4c5 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 16 Mar 2020 14:53:20 +0100
Subject: [PATCH 17/51] checkstyle

---
 .../compat/version7/RestIndexActionV7.java    | 19 +++++++++++++++++++
 .../rest/compat/AbstractCompatRestTest.java   |  1 -
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
index f159801a3ce94..62af68b424342 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package org.elasticsearch.rest.compat.version7;
 
 import org.apache.logging.log4j.LogManager;
diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index 94b28abc5017e..9cf2a780471fc 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.test.rest.yaml.section.DoSection;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 
-import java.net.http.HttpHeaders;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;

From a0fce895fa6388c05c668c8d56ca3cc93ab24650 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 16 Mar 2020 15:42:25 +0100
Subject: [PATCH 18/51] Revert "Merge branch 'master' into
 compat/type-index-get"

This reverts commit ae1799ff9f5ca11da20105ce5f9f814aa9b61b21, reversing
changes made to f00f438b8c66a39938a24fc99b523787229be4c5.
---
 .ci/matrix-runtime-javas.yml                  |   1 +
 CONTRIBUTING.md                               |   8 +-
 build.gradle                                  |   3 +-
 .../gradle/DistributionDownloadPlugin.java    |  23 +-
 .../gradle/JdkDownloadPlugin.java             |  33 +-
 buildSrc/version.properties                   |   4 +-
 .../client/analytics/ParsedTopMetrics.java    |  12 +-
 .../client/cluster/ProxyModeInfo.java         |   6 +-
 .../client/cluster/RemoteConnectionInfo.java  |   6 +-
 .../client/ml/dataframe/Classification.java   |  81 +-
 .../client/ml/dataframe/Regression.java       |  30 +-
 .../ml/inference/TrainedModelConfig.java      |  27 +-
 .../ml/inference/TrainedModelDefinition.java  |   8 +-
 .../trainedmodel/ensemble/Ensemble.java       |   6 +-
 .../java/org/elasticsearch/client/EqlIT.java  | 133 +--
 .../client/MachineLearningIT.java             |   8 +-
 .../client/analytics/AnalyticsAggsIT.java     |  55 +-
 .../MlClientDocumentationIT.java              |  10 +-
 .../ml/dataframe/ClassificationTests.java     |   3 +-
 .../client/ml/dataframe/RegressionTests.java  |   2 +-
 .../ml/inference/TrainedModelConfigTests.java |   7 +-
 distribution/docker/docker-compose.yml        |   2 +-
 docs/Versions.asciidoc                        |   4 +-
 .../ml/put-data-frame-analytics.asciidoc      |   3 +-
 docs/plugins/integrations.asciidoc            |   4 +-
 docs/python/index.asciidoc                    |   4 +-
 .../bucket/geohashgrid-aggregation.asciidoc   |   2 +-
 .../metrics/top-metrics-aggregation.asciidoc  |  35 +-
 .../flatten-graph-tokenfilter.asciidoc        |   2 +
 .../word-delimiter-graph-tokenfilter.asciidoc | 551 ++---------
 .../word-delimiter-tokenfilter.asciidoc       | 421 ++-------
 .../simplepattern-tokenizer.asciidoc          |   2 +
 .../simplepatternsplit-tokenizer.asciidoc     |   2 +
 docs/reference/cluster/remote-info.asciidoc   |  36 +-
 docs/reference/cluster/stats.asciidoc         | 299 +-----
 docs/reference/eql/search.asciidoc            |   9 -
 docs/reference/glossary.asciidoc              | 192 +---
 .../reference/ilm/policy-definitions.asciidoc |   2 +-
 .../images/analysis/token-graph-basic.svg     |  45 -
 .../images/analysis/token-graph-wd.svg        |  52 --
 .../images/analysis/token-graph-wdg.svg       |  53 --
 .../apis/enrich/put-enrich-policy.asciidoc    |  15 +-
 docs/reference/ingest/enrich.asciidoc         |  20 +-
 .../ingest/processors/inference.asciidoc      |   4 +-
 docs/reference/mapping/types.asciidoc         |   4 +-
 .../mapping/types/histogram.asciidoc          |   1 +
 .../mapping/types/search-as-you-type.asciidoc |  15 +-
 .../reference/mapping/types/wildcard.asciidoc |  53 --
 .../migration/migrate_8_0/snapshots.asciidoc  |   7 -
 .../apis/put-dfanalytics.asciidoc             |  12 +-
 docs/reference/ml/ml-shared.asciidoc          |  28 +-
 .../modules/cross-cluster-search.asciidoc     |  37 +-
 .../modules/remote-clusters.asciidoc          | 156 +---
 docs/reference/redirects.asciidoc             |  10 -
 .../search/request/stored-fields.asciidoc     |   7 +-
 .../sql/language/data-types.asciidoc          |  31 +-
 .../apis/get-transform-stats.asciidoc         |   4 -
 .../common/xcontent/AbstractObjectParser.java |  25 -
 .../xcontent/ConstructingObjectParser.java    |  84 +-
 .../common/xcontent/ObjectParser.java         |  28 +-
 .../common/xcontent/ObjectParserTests.java    |  73 +-
 modules/ingest-geoip/build.gradle             |   5 +-
 .../jackson-databind-2.8.11.4.jar.sha1        |   1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |   1 -
 ...ssions-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...ssions-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 .../mustache/RestSearchTemplateAction.java    |   1 +
 .../TransportSearchTemplateAction.java        |  21 -
 .../test/lang_mustache/30_search_template.yml |  13 -
 .../50_multi_search_template.yml              |   8 -
 .../org/elasticsearch/painless/Operation.java |  58 +-
 .../painless/node/AExpression.java            | 102 +--
 .../painless/node/AStatement.java             | 131 ++-
 .../painless/node/AStoreable.java             |  19 +-
 .../painless/node/EAssignment.java            | 198 ++--
 .../elasticsearch/painless/node/EBinary.java  | 450 +++++++--
 .../elasticsearch/painless/node/EBool.java    |  21 +-
 .../elasticsearch/painless/node/EBoolean.java |  13 +-
 .../painless/node/ECallLocal.java             |  26 +-
 .../painless/node/ECapturingFunctionRef.java  |  17 +-
 .../elasticsearch/painless/node/EComp.java    | 243 ++++-
 .../painless/node/EConditional.java           |  52 +-
 .../painless/node/EConstant.java              |  32 +-
 .../elasticsearch/painless/node/EDecimal.java |  15 +-
 .../elasticsearch/painless/node/EElvis.java   |  43 +-
 .../painless/node/EExplicit.java              |  18 +-
 .../painless/node/EFunctionRef.java           |  18 +-
 .../painless/node/EInstanceof.java            |  25 +-
 .../elasticsearch/painless/node/ELambda.java  |  34 +-
 .../painless/node/EListInit.java              |  28 +-
 .../elasticsearch/painless/node/EMapInit.java |  35 +-
 .../painless/node/ENewArray.java              |  20 +-
 .../painless/node/ENewArrayFunctionRef.java   |  17 +-
 .../elasticsearch/painless/node/ENewObj.java  |  30 +-
 .../elasticsearch/painless/node/ENull.java    |  21 +-
 .../elasticsearch/painless/node/ENumeric.java |  27 +-
 .../elasticsearch/painless/node/ERegex.java   |  12 +-
 .../elasticsearch/painless/node/EStatic.java  |  13 +-
 .../elasticsearch/painless/node/EString.java  |  13 +-
 .../elasticsearch/painless/node/EUnary.java   | 111 ++-
 .../painless/node/EVariable.java              |  24 +-
 .../elasticsearch/painless/node/PBrace.java   |  55 +-
 .../painless/node/PCallInvoke.java            |  34 +-
 .../elasticsearch/painless/node/PField.java   |  68 +-
 .../painless/node/PSubArrayLength.java        |  13 +-
 .../painless/node/PSubBrace.java              |  16 +-
 .../painless/node/PSubCallInvoke.java         |  20 +-
 .../painless/node/PSubDefArray.java           |  17 +-
 .../painless/node/PSubDefCall.java            |  22 +-
 .../painless/node/PSubDefField.java           |  13 +-
 .../painless/node/PSubField.java              |  13 +-
 .../painless/node/PSubListShortcut.java       |  18 +-
 .../painless/node/PSubMapShortcut.java        |  18 +-
 .../painless/node/PSubNullSafeCallInvoke.java |  15 +-
 .../painless/node/PSubNullSafeField.java      |  20 +-
 .../painless/node/PSubShortcut.java           |  13 +-
 .../elasticsearch/painless/node/SBlock.java   |  36 +-
 .../elasticsearch/painless/node/SBreak.java   |  17 +-
 .../elasticsearch/painless/node/SCatch.java   |  32 +-
 .../painless/node/SContinue.java              |  17 +-
 .../painless/node/SDeclBlock.java             |  11 +-
 .../painless/node/SDeclaration.java           |  13 +-
 .../org/elasticsearch/painless/node/SDo.java  |  29 +-
 .../elasticsearch/painless/node/SEach.java    |  32 +-
 .../painless/node/SExpression.java            |  30 +-
 .../org/elasticsearch/painless/node/SFor.java |  52 +-
 .../painless/node/SFunction.java              |   9 +-
 .../org/elasticsearch/painless/node/SIf.java  |  27 +-
 .../elasticsearch/painless/node/SIfElse.java  |  48 +-
 .../elasticsearch/painless/node/SReturn.java  |  22 +-
 .../painless/node/SSubEachArray.java          |  11 +-
 .../painless/node/SSubEachIterable.java       |  13 +-
 .../elasticsearch/painless/node/SThrow.java   |  20 +-
 .../org/elasticsearch/painless/node/STry.java |  49 +-
 .../elasticsearch/painless/node/SWhile.java   |  31 +-
 .../percolator/QueryAnalyzerTests.java        |  13 +-
 ...rs-icu-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...rs-icu-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...romoji-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...romoji-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...s-nori-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...s-nori-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...onetic-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...onetic-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...martcn-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...martcn-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...tempel-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...tempel-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...ologik-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...ologik-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 .../jackson-databind-2.8.11.4.jar.sha1        |   1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |   1 -
 .../jackson-databind-2.8.11.4.jar.sha1        |   1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |   1 -
 .../index/engine/EvilInternalEngineTests.java |   3 +-
 .../test/multi_cluster/20_info.yml            |   6 +-
 .../rest-api-spec/api/cat.aliases.json        |  12 -
 .../rest-api-spec/api/cat.indices.json        |  12 -
 .../rest-api-spec/api/cluster.health.json     |   1 -
 .../rest-api-spec/api/cluster.state.json      |   1 -
 .../resources/rest-api-spec/api/count.json    |   1 -
 .../rest-api-spec/api/delete_by_query.json    |   1 -
 .../rest-api-spec/api/field_caps.json         |   1 -
 .../api/indices.clear_cache.json              |   1 -
 .../rest-api-spec/api/indices.close.json      |   1 -
 .../rest-api-spec/api/indices.delete.json     |   1 -
 .../rest-api-spec/api/indices.exists.json     |   1 -
 .../api/indices.exists_alias.json             |   1 -
 .../api/indices.exists_type.json              |   1 -
 .../rest-api-spec/api/indices.flush.json      |   1 -
 .../rest-api-spec/api/indices.forcemerge.json |   1 -
 .../rest-api-spec/api/indices.get.json        |   1 -
 .../rest-api-spec/api/indices.get_alias.json  |   3 +-
 .../api/indices.get_field_mapping.json        |   1 -
 .../api/indices.get_mapping.json              |   1 -
 .../api/indices.get_settings.json             |   1 -
 .../api/indices.get_upgrade.json              |   1 -
 .../rest-api-spec/api/indices.open.json       |   1 -
 .../api/indices.put_mapping.json              |   1 -
 .../api/indices.put_settings.json             |   1 -
 .../rest-api-spec/api/indices.refresh.json    |   1 -
 .../rest-api-spec/api/indices.segments.json   |   1 -
 .../api/indices.shard_stores.json             |   1 -
 .../rest-api-spec/api/indices.stats.json      |   1 -
 .../rest-api-spec/api/indices.upgrade.json    |   1 -
 .../api/indices.validate_query.json           |   1 -
 .../rest-api-spec/api/rank_eval.json          |   1 -
 .../resources/rest-api-spec/api/search.json   |   1 -
 .../rest-api-spec/api/search_shards.json      |   1 -
 .../rest-api-spec/api/search_template.json    |   1 -
 .../rest-api-spec/api/update_by_query.json    |   1 -
 .../test/cat.aliases/40_hidden.yml            | 150 ---
 .../test/cat.indices/20_hidden.yml            | 243 -----
 .../test/search.aggregation/230_composite.yml | 137 ---
 .../test/search.aggregation/320_missing.yml   |  98 --
 .../search.highlight/40_keyword_ignore.yml    |  64 --
 .../test/search/320_disallow_queries.yml      | 185 +++-
 ...common-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...common-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...codecs-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...codecs-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...e-core-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...e-core-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...ouping-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...ouping-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...ighter-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...ighter-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...e-join-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...e-join-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...memory-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...memory-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...e-misc-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...e-misc-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...ueries-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...ueries-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...parser-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...parser-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...andbox-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...andbox-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...extras-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...extras-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...tial3d-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...tial3d-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 ...uggest-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...uggest-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 .../org/apache/lucene/queries/XIntervals.java | 861 ++++++++++++++++++
 .../cluster/node/info/NodesInfoRequest.java   |   2 +-
 .../cluster/node/stats/NodesStatsRequest.java | 200 ++--
 .../restore/RestoreSnapshotRequest.java       |  98 +-
 .../RestoreSnapshotRequestBuilder.java        |  54 ++
 .../indices/forcemerge/ForceMergeRequest.java |  30 -
 .../action/fieldcaps/FieldCapabilities.java   |   4 +-
 .../ingest/SimulateExecutionService.java      |   7 -
 .../ingest/SimulatePipelineRequest.java       |   9 +-
 .../search/AbstractSearchAsyncAction.java     |  47 +-
 .../search/CanMatchPreFilterSearchPhase.java  |  34 +-
 .../action/search/ClearScrollController.java  |   2 +-
 .../action/search/DfsQueryPhase.java          |  12 +-
 .../action/search/FetchSearchPhase.java       |  37 +-
 .../action/search/ScrollIdForNode.java        |  13 +-
 .../SearchDfsQueryThenFetchAsyncAction.java   |  24 +-
 .../action/search/SearchPhaseContext.java     |   7 +-
 .../action/search/SearchPhaseController.java  |  13 +-
 .../action/search/SearchProgressListener.java |  51 +-
 .../SearchQueryThenFetchAsyncAction.java      |  28 +-
 .../action/search/SearchRequest.java          |   7 +-
 .../action/search/SearchRequestBuilder.java   |   2 +-
 .../action/search/SearchResponse.java         |   4 -
 .../search/SearchScrollAsyncAction.java       |   9 +-
 ...SearchScrollQueryThenFetchAsyncAction.java |   4 +-
 .../action/search/SearchShard.java            |   8 +-
 .../action/search/SearchShardTask.java        |   1 +
 .../action/search/SearchTask.java             |   4 +-
 .../action/search/SearchTransportService.java |  30 +-
 .../action/search/TransportSearchAction.java  |  12 +-
 .../action/search/TransportSearchHelper.java  |  30 +-
 .../elasticsearch/client/node/NodeClient.java |  38 +
 .../elasticsearch/cluster/ClusterModule.java  |   5 -
 .../cluster/NodeConnectionsService.java       |  38 +-
 .../cluster/coordination/Coordinator.java     |   4 +-
 .../cluster/coordination/JoinHelper.java      |   6 +-
 .../cluster/coordination/JoinRequest.java     |  46 +-
 .../cluster/metadata/ComponentTemplate.java   | 300 ------
 .../metadata/ComponentTemplateMetadata.java   | 169 ----
 .../cluster/metadata/MetaData.java            |  35 -
 .../cluster/node/DiscoveryNodes.java          |  19 +-
 .../command/AllocationCommands.java           |   3 +
 .../SpanBooleanQueryRewriteWithMaxClause.java |   6 +-
 .../elasticsearch/common/util/BitArray.java   |  32 +-
 .../gateway/GatewayAllocator.java             |   1 -
 .../elasticsearch/index/engine/Engine.java    |  11 +-
 .../index/engine/InternalEngine.java          |  30 +-
 .../index/engine/NoOpEngine.java              |  19 +-
 .../index/engine/ReadOnlyEngine.java          |  37 +-
 .../plain/BinaryDVIndexFieldData.java         |  17 +-
 .../index/mapper/DateFieldMapper.java         |  19 -
 .../index/mapper/KeywordFieldMapper.java      |   4 +-
 .../index/mapper/ParseContext.java            |  18 +
 .../index/mapper/StringFieldType.java         |  44 +-
 .../index/mapper/TextFieldMapper.java         |   7 +-
 .../index/mapper/TypeFieldMapper.java         |  61 +-
 .../index/query/IntervalsSourceProvider.java  |   8 +-
 .../elasticsearch/index/shard/IndexShard.java |   4 +-
 .../index/shard/StoreRecovery.java            |   3 +-
 .../index/termvectors/TermVectorsService.java |  23 +-
 .../rest/action/cat/RestAliasAction.java      |   2 -
 .../rest/action/cat/RestIndicesAction.java    |  14 +-
 .../rest/action/search/RestSearchAction.java  |   2 +-
 .../search/DefaultSearchContext.java          |   7 +-
 .../elasticsearch/search/DocValueFormat.java  |  15 -
 .../search/SearchContextMissingException.java |  17 +-
 .../search/SearchPhaseResult.java             |   9 +-
 .../elasticsearch/search/SearchService.java   |  96 +-
 .../aggregations/InternalAggregation.java     |  24 -
 .../aggregations/InternalAggregations.java    |   9 -
 .../InternalMultiBucketAggregation.java       |  19 +-
 .../InternalSingleBucketAggregation.java      |  10 -
 .../adjacency/InternalAdjacencyMatrix.java    |   3 +-
 .../bucket/composite/CompositeAggregator.java |   3 +-
 .../bucket/composite/LongValuesSource.java    |   3 +-
 .../bucket/terms/InternalTerms.java           |  58 +-
 .../aggregations/metrics/MinAggregator.java   |   9 +-
 .../pipeline/SiblingPipelineAggregator.java   |  42 +-
 .../search/dfs/DfsSearchResult.java           |   9 +-
 .../search/fetch/FetchSearchResult.java       |   9 +-
 .../search/fetch/QueryFetchSearchResult.java  |   5 +-
 .../search/fetch/ShardFetchRequest.java       |  17 +-
 .../search/fetch/ShardFetchSearchRequest.java |   3 +-
 .../subphase/highlight/PlainHighlighter.java  |  13 +-
 .../highlight/UnifiedHighlighter.java         |  14 +-
 .../internal/FilteredSearchContext.java       |   2 +-
 .../internal/InternalScrollSearchRequest.java |  16 +-
 .../search/internal/SearchContext.java        |   2 +-
 .../search/internal/SearchContextId.java      |  82 --
 .../search/query/QuerySearchRequest.java      |  24 +-
 .../search/query/QuerySearchResult.java       |  13 +-
 .../elasticsearch/search/sort/MinAndMax.java  |   4 +-
 .../elasticsearch/search/sort/SortValue.java  |  15 -
 .../java/org/elasticsearch/tasks/Task.java    |   7 -
 .../elasticsearch/threadpool/ThreadPool.java  |   3 +
 .../transport/ProxyConnectionStrategy.java    |   6 +-
 .../ElasticsearchExceptionTests.java          |   5 +-
 .../ExceptionSerializationTests.java          |  14 +-
 .../node/stats/NodesStatsRequestTests.java    | 149 ---
 .../restore/RestoreSnapshotRequestTests.java  |  10 +
 .../indices/forcemerge/ForceMergeIT.java      |  89 --
 .../SimulatePipelineRequestParsingTests.java  |  31 -
 .../AbstractSearchAsyncActionTests.java       |  30 +-
 .../CanMatchPreFilterSearchPhaseTests.java    |  86 +-
 .../search/ClearScrollControllerTests.java    |  28 +-
 .../action/search/CountedCollectorTests.java  |   6 +-
 .../action/search/DfsQueryPhaseTests.java     |  60 +-
 .../action/search/FetchSearchPhaseTests.java  |  71 +-
 .../action/search/MockSearchPhaseContext.java |   5 +-
 .../action/search/SearchAsyncActionTests.java |  41 +-
 .../search/SearchPhaseControllerTests.java    |  56 +-
 .../SearchProgressActionListenerIT.java       |  21 +-
 .../search/SearchScrollAsyncActionTests.java  |  72 +-
 .../search/SearchScrollRequestTests.java      |   7 +-
 .../action/search/SearchShardTests.java       |  83 --
 .../search/TransportSearchHelperTests.java    |  36 +-
 .../cluster/NodeConnectionsServiceTests.java  | 119 +--
 .../cluster/coordination/JoinHelperTests.java |  10 +-
 .../cluster/coordination/MessagesTests.java   |  11 +-
 .../cluster/coordination/NodeJoinTests.java   |  52 +-
 .../cluster/coordination/ZenDiscoveryIT.java  |   2 +-
 .../ComponentTemplateMetadataTests.java       |  70 --
 .../metadata/ComponentTemplateTests.java      | 163 ----
 .../cluster/metadata/MetaDataTests.java       |  36 +-
 .../metadata/ToAndFromJsonMetaDataTests.java  |  16 -
 .../common/util/BitArrayTests.java            |  36 -
 .../ReplicaShardAllocatorSyncIdIT.java        |   3 +-
 .../index/engine/InternalEngineTests.java     |  35 +-
 .../index/engine/NoOpEngineTests.java         |   9 +-
 .../index/mapper/DateFieldMapperTests.java    |   3 +-
 .../index/mapper/DocumentParserTests.java     |  14 +-
 .../mapper/FieldNamesFieldMapperTests.java    |   3 +-
 .../index/mapper/IpFieldMapperTests.java      |   3 +-
 .../index/mapper/KeywordFieldMapperTests.java |   4 -
 .../index/mapper/NumberFieldMapperTests.java  |   4 +-
 .../index/mapper/TypeFieldTypeTests.java      |  37 +-
 .../query/IntervalQueryBuilderTests.java      |  15 +-
 .../index/query/RangeQueryBuilderTests.java   |   7 +
 .../query/WildcardQueryBuilderTests.java      |  11 +-
 .../index/shard/IndexShardTests.java          |  18 -
 .../search/DefaultSearchContextTests.java     |  18 +-
 .../search/SearchServiceTests.java            |  84 +-
 .../search/aggregations/bucket/MissingIT.java | 184 ++++
 .../composite/CompositeAggregatorTests.java   |  76 +-
 .../geogrid/GeoGridAggregatorTestCase.java    |  13 -
 .../missing/MissingAggregatorTests.java       | 687 +++-----------
 .../search/query/QuerySearchResultTests.java  |   6 +-
 .../search/query/SearchQueryIT.java           | 138 +--
 .../search/scroll/SearchScrollIT.java         |  40 -
 .../snapshots/SnapshotRequestsTests.java      |   2 +
 .../RemoteClusterConnectionTests.java         |   4 +-
 .../org/elasticsearch/test/ESTestCase.java    |   6 -
 .../elasticsearch/test/TestSearchContext.java |   5 +-
 .../elasticsearch/test/XContentTestUtils.java |   7 -
 .../test/rest/ESRestTestCase.java             |   1 -
 .../authentication/custom-realm.asciidoc      |   5 +-
 .../authentication/saml-guide.asciidoc        |   2 +-
 .../topmetrics/InternalTopMetrics.java        |  97 +-
 .../topmetrics/TopMetricsAggregator.java      | 309 +------
 .../TopMetricsAggregatorFactory.java          |   8 +-
 .../InternalTopMetricsReduceTests.java        |   4 +-
 .../topmetrics/InternalTopMetricsTests.java   | 138 +--
 .../TopMetricsAggregatorMetricsTests.java     | 213 -----
 .../topmetrics/TopMetricsAggregatorTests.java |  70 +-
 x-pack/plugin/async-search/build.gradle       |  42 -
 x-pack/plugin/async-search/qa/build.gradle    |   8 -
 .../async-search/qa/security/build.gradle     |  19 -
 .../plugin/async-search/qa/security/roles.yml |  33 -
 .../xpack/search/AsyncSearchSecurityIT.java   | 162 ----
 .../xpack/search/AsyncSearch.java             |  92 --
 .../xpack/search/AsyncSearchId.java           | 106 ---
 .../xpack/search/AsyncSearchIndexService.java | 342 -------
 .../search/AsyncSearchMaintenanceService.java | 117 ---
 .../xpack/search/AsyncSearchTask.java         | 370 --------
 .../xpack/search/MutableSearchResponse.java   | 173 ----
 .../search/RestDeleteAsyncSearchAction.java   |  40 -
 .../search/RestGetAsyncSearchAction.java      |  50 -
 .../search/RestSubmitAsyncSearchAction.java   |  71 --
 .../TransportDeleteAsyncSearchAction.java     |  72 --
 .../search/TransportGetAsyncSearchAction.java | 144 ---
 .../TransportSubmitAsyncSearchAction.java     | 201 ----
 .../plugin-metadata/plugin-security.policy    |   0
 .../xpack/search/AsyncSearchActionTests.java  | 256 ------
 .../xpack/search/AsyncSearchIdTests.java      |  42 -
 .../search/AsyncSearchIndexServiceTests.java  | 103 ---
 .../search/AsyncSearchIntegTestCase.java      | 394 --------
 .../search/AsyncSearchResponseTests.java      | 131 ---
 .../xpack/search/AsyncSearchTaskTests.java    | 182 ----
 .../search/DeleteAsyncSearchRequestTests.java |  24 -
 .../search/GetAsyncSearchRequestTests.java    |  41 -
 .../search/SubmitAsyncSearchRequestTests.java | 112 ---
 x-pack/plugin/build.gradle                    |   2 -
 .../xpack/ccr/action/ShardChangesAction.java  |   2 +-
 .../ccr/action/ShardFollowTasksExecutor.java  |  46 +-
 .../elasticsearch/xpack/CcrIntegTestCase.java |   1 +
 .../xpack/CcrSingleNodeTestCase.java          |   1 +
 .../license/XPackLicenseState.java            |  10 -
 .../xpack/core/XPackClientPlugin.java         |  12 +-
 .../xpack/core/XPackSettings.java             |   6 +-
 .../core/beats/BeatsFeatureSetUsage.java      |  24 +
 .../xpack/core/ccr/AutoFollowStats.java       |   5 +-
 .../xpack/core/ilm/FreezeStep.java            |   8 +-
 .../ml/action/EstimateModelMemoryAction.java  | 176 ----
 .../dataframe/analyses/BoostedTreeParams.java |  40 +-
 .../ml/dataframe/analyses/Classification.java |  67 +-
 .../dataframe/analyses/DataFrameAnalysis.java |  28 +-
 .../dataframe/analyses/OutlierDetection.java  |   2 +-
 .../ml/dataframe/analyses/Regression.java     |   2 +-
 .../ml/dataframe/evaluation/Evaluation.java   |   5 +-
 .../evaluation/EvaluationMetric.java          |   5 +-
 .../evaluation/EvaluationParameters.java      |  25 -
 .../evaluation/classification/Accuracy.java   |  19 +-
 .../MulticlassConfusionMatrix.java            |  61 +-
 .../classification/PainlessScripts.java       |  35 -
 .../evaluation/classification/Precision.java  |  14 +-
 .../evaluation/classification/Recall.java     |  16 +-
 .../regression/MeanSquaredError.java          |   5 +-
 .../evaluation/regression/RSquared.java       |   5 +-
 .../AbstractConfusionMatrixMetric.java        |   5 +-
 .../evaluation/softclassification/AucRoc.java |   5 +-
 .../core/ml/inference/TrainedModelConfig.java |  44 +-
 .../ml/inference/TrainedModelDefinition.java  |   3 +-
 .../trainedmodel/ensemble/Ensemble.java       |  11 +-
 .../persistence/AnomalyDetectorsIndex.java    |  39 +-
 .../ml/job/results/ReservedFieldNames.java    |   3 +-
 .../xpack/core/ml/utils/MlIndexAndAlias.java  | 167 ++--
 .../search/action/AsyncSearchResponse.java    | 208 -----
 .../action/DeleteAsyncSearchAction.java       |  72 --
 .../search/action/GetAsyncSearchAction.java   | 120 ---
 .../action/SubmitAsyncSearchAction.java       |  17 -
 .../action/SubmitAsyncSearchRequest.java      | 183 ----
 .../action/CreateApiKeyRequestBuilder.java    |   6 +-
 .../security/action/GrantApiKeyAction.java    |  24 -
 .../security/action/GrantApiKeyRequest.java   | 166 ----
 .../index/RestrictedIndicesNames.java         |   2 +-
 .../core/slm/SnapshotLifecyclePolicy.java     |   4 +-
 .../transforms/pivot/SingleGroupSource.java   |   4 +-
 .../state_index_template.json                 |   3 +-
 .../xpack/core/ml/config_index_mappings.json  |   7 +-
 .../core/ml/inference_index_template.json     |   3 -
 .../xpack/core/ml/size_based_ilm_policy.json  |  11 -
 .../xpack/core/ml/stats_index_template.json   |   4 +-
 .../snapshots/SourceOnlySnapshotIT.java       |  12 +-
 .../xpack/core/ilm/FreezeStepTests.java       |  31 +-
 .../DataFrameAnalyticsConfigTests.java        |   2 -
 .../analyses/BoostedTreeParamsTests.java      |  10 +-
 .../analyses/ClassificationTests.java         | 101 +-
 .../evaluation/EvaluationParametersTests.java |  18 -
 .../classification/AccuracyTests.java         |  13 +-
 .../classification/ClassificationTests.java   |   9 +-
 .../MulticlassConfusionMatrixTests.java       | 127 +--
 .../classification/PainlessScriptsTests.java  |  19 -
 .../classification/PrecisionTests.java        |   9 +-
 .../classification/RecallTests.java           |   9 +-
 .../regression/RegressionTests.java           |   5 +-
 .../BinarySoftClassificationTests.java        |   5 +-
 .../ml/inference/TrainedModelConfigTests.java |  14 +-
 .../core/ml/utils/MlIndexAndAliasTests.java   |  72 +-
 .../deprecation/NodeDeprecationChecks.java    |  30 -
 .../NodeDeprecationChecksTests.java           |  44 -
 .../test/eql/CommonEqlRestTestCase.java       |  74 +-
 .../xpack/eql/optimizer/Optimizer.java        |  27 -
 .../xpack/eql/action/EqlActionIT.java         |  71 +-
 .../xpack/eql/optimizer/OptimizerTests.java   |  59 +-
 .../planner/AbstractQueryFolderTestCase.java  |  38 -
 .../eql/planner/QueryFolderFailTests.java     |  25 -
 .../xpack/eql/planner/QueryFolderOkTests.java | 105 ---
 .../xpack/eql/planner/QueryFolderTests.java   |  56 ++
 .../resources/test_queries_unsupported.toml   | 119 ++-
 .../index/engine/FrozenEngine.java            |  23 +-
 .../ilm/TimeSeriesLifecycleActionsIT.java     |   5 +-
 .../xpack/slm/SnapshotLifecycleRestIT.java    |   1 -
 .../IndexLifecycleInitialisationTests.java    |   1 +
 .../SnapshotLifecycleInitialisationTests.java |   1 +
 .../slm/SnapshotLifecyclePolicyTests.java     |  13 -
 .../xpack/logstash/Logstash.java              |  25 +-
 .../logstash/LogstashInfoTransportAction.java |   8 +-
 .../LogstashUsageTransportAction.java         |  33 +-
 .../LogstashInfoTransportActionTests.java     |  38 +-
 x-pack/plugin/ml/build.gradle                 |   1 -
 .../ml/qa/ml-with-security/build.gradle       |  11 +-
 .../ml/integration/CategorizationIT.java      |   3 +-
 .../ClassificationEvaluationIT.java           | 227 +----
 .../ml/integration/ClassificationIT.java      |  30 +-
 .../ml/integration/DeleteExpiredDataIT.java   |  16 +-
 .../ml/integration/InferenceIngestIT.java     |  47 +-
 .../ml/integration/MlNativeIntegTestCase.java |  13 +-
 .../xpack/ml/integration/RegressionIT.java    |  21 +-
 .../xpack/ml/MachineLearning.java             |   5 -
 .../MachineLearningUsageTransportAction.java  |   4 -
 .../xpack/ml/MlIndexTemplateRegistry.java     |  61 +-
 .../TransportEstimateModelMemoryAction.java   | 193 ----
 .../TransportEvaluateDataFrameAction.java     |  32 +-
 ...ransportStartDataFrameAnalyticsAction.java |  45 +-
 .../scroll/TimeBasedExtractedFields.java      |   3 +-
 .../extractor/DataFrameDataExtractor.java     |   4 -
 .../extractor/ExtractedFieldsDetector.java    |  17 +-
 .../process/AnalyticsProcessConfig.java       |  30 +-
 .../process/AnalyticsProcessManager.java      |   2 +-
 .../process/AnalyticsResultProcessor.java     |  13 +-
 .../xpack/ml/extractor/ExtractedFields.java   |  14 +-
 .../xpack/ml/extractor/MultiField.java        |   2 +-
 .../inference/ingest/InferenceProcessor.java  |  30 +-
 .../inference/loadingservice/LocalModel.java  |   9 +-
 .../ml/inference/loadingservice/Model.java    |  24 -
 .../loadingservice/ModelLoadingService.java   |   6 +-
 .../job/RestEstimateModelMemoryAction.java    |  39 -
 .../MachineLearningLicensingTests.java        |   2 +-
 .../ml/MlIndexTemplateRegistryTests.java      | 181 ----
 .../xpack/ml/MlSingleNodeTestCase.java        |   9 +-
 ...ansportEstimateModelMemoryActionTests.java | 131 ---
 ...sportGetTrainedModelsStatsActionTests.java |   2 +-
 .../DataFrameDataExtractorTests.java          |   6 +-
 .../ExtractedFieldsDetectorTests.java         |   4 +-
 .../process/AnalyticsProcessConfigTests.java  | 170 ----
 .../process/AnalyticsProcessManagerTests.java |   4 +-
 .../AnalyticsResultProcessorTests.java        |  25 +-
 .../ml/extractor/ExtractedFieldsTests.java    |   8 +-
 .../InferenceProcessorFactoryTests.java       |  18 +-
 .../loadingservice/LocalModelTests.java       |  24 +-
 .../ml/integration/AnnotationIndexIT.java     |   8 +
 .../AutodetectResultProcessorIT.java          |   7 +-
 .../integration/BasicDistributedJobsIT.java   |   2 +-
 .../AutodetectProcessManagerTests.java        |  30 +-
 .../xpack/ml/support/BaseMlIntegTestCase.java |  11 +-
 .../extractor/AbstractFieldHitExtractor.java  |   2 -
 .../ql/expression/predicate/regex/Like.java   |  13 +
 .../predicate/regex/LikePattern.java          |   8 +-
 .../ql/expression/predicate/regex/RLike.java  |  17 +-
 .../predicate/regex/RLikePattern.java         |  20 -
 .../predicate/regex/RegexMatch.java           |  30 +-
 .../predicate/regex/StringPattern.java        |  13 -
 .../xpack/ql/index/IndexResolver.java         |  21 +-
 .../ql/planner/ExpressionTranslators.java     |   4 +-
 .../xpack/ql/type/ConstantKeywordEsField.java |  22 -
 .../xpack/ql/type/DataTypeConverter.java      |   8 +-
 .../xpack/ql/type/DataTypes.java              |  38 +-
 .../xpack/ql/type/KeywordEsField.java         |   7 +-
 .../xpack/ql/type/TextEsField.java            |   3 +-
 .../elasticsearch/xpack/ql/type/Types.java    |   3 -
 .../xpack/ql/expression/LiteralTests.java     |   3 +-
 .../ql/optimizer/OptimizerRulesTests.java     |   5 +-
 .../ql/type/DataTypeConversionTests.java      |   7 -
 .../xpack/ql/type/TypesTests.java             |  14 +-
 .../resources/mapping-constant-keyword.json   |   8 -
 .../test/resources/mapping-multi-field.json   |   4 -
 .../security/SecurityInBasicRestTestCase.java |  49 -
 .../security/SecurityWithBasicLicenseIT.java  |  23 +-
 .../security/qa/security-trial/build.gradle   |  28 -
 .../SecurityOnTrialLicenseRestTestCase.java   | 111 ---
 .../xpack/security/apikey/ApiKeyRestIT.java   | 118 ---
 .../src/test/resources/roles.yml              |   8 -
 .../xpack/security/Security.java              |   5 -
 .../action/TransportCreateApiKeyAction.java   |  32 +-
 .../action/TransportGrantApiKeyAction.java    |  88 --
 .../xpack/security/authc/ApiKeyService.java   |   2 +-
 .../xpack/security/authc/TokenService.java    |  29 -
 .../authc/support/ApiKeyGenerator.java        |  58 --
 .../xpack/security/authz/RBACEngine.java      |  49 +-
 .../SecuritySearchOperationListener.java      |   3 +-
 .../action/apikey/RestGrantApiKeyAction.java  |  93 --
 .../TransportGrantApiKeyActionTests.java      | 230 -----
 .../security/authc/TokenServiceMock.java      |  70 --
 .../authc/support/ApiKeyGeneratorTests.java   |  86 --
 .../SecuritySearchOperationListenerTests.java |  30 +-
 .../xpack/security/test/SecurityMocks.java    |  28 -
 .../xpack/sql/jdbc/EsDataSource.java          |   7 +-
 .../xpack/sql/jdbc/EsDriver.java              |   8 +-
 .../elasticsearch/xpack/sql/jdbc/EsType.java  |   1 -
 .../xpack/sql/jdbc/InfoResponse.java          |  19 +-
 .../xpack/sql/jdbc/JdbcConfiguration.java     |   4 +-
 .../xpack/sql/jdbc/JdbcConnection.java        |   4 +-
 .../xpack/sql/jdbc/JdbcDatabaseMetaData.java  |  14 +-
 .../xpack/sql/jdbc/JdbcHttpClient.java        |  18 +-
 .../xpack/sql/jdbc/TypeConverter.java         |   8 -
 .../xpack/sql/jdbc/TypeUtils.java             |   3 +-
 .../xpack/sql/jdbc/VersionParityTests.java    |   7 +-
 .../xpack/sql/jdbc/VersionTests.java          |   9 +-
 .../sql/qa/multi_node/RestSqlMultinodeIT.java |   3 +-
 .../sql/qa/security/RestSqlSecurityIT.java    |  22 +-
 .../xpack/sql/qa/security/UserFunctionIT.java |   4 +-
 .../xpack/sql/qa/single_node/RestSqlIT.java   |   8 +-
 .../xpack/sql/qa/FieldExtractorTestCase.java  |  34 -
 .../xpack/sql/qa/SqlProtocolTestCase.java     |  18 +-
 .../xpack/sql/qa/jdbc/DataLoader.java         |   9 +-
 .../sql/qa/rest/BaseRestSqlTestCase.java      |  11 +-
 .../xpack/sql/qa/rest/RestSqlTestCase.java    |  38 +-
 .../sql/qa/rest/RestSqlUsageTestCase.java     |  20 +-
 .../sql/qa/src/main/resources/alias.csv-spec  |   6 +-
 .../qa/src/main/resources/command.csv-spec    |   6 +-
 .../main/resources/constant-keyword.csv-spec  | 163 ----
 .../single-node-only/command-sys.csv-spec     | 148 ++-
 ...e-core-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...e-core-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 +
 .../sql/action/AbstractSqlQueryRequest.java   |  32 +-
 .../xpack/sql/action/AbstractSqlRequest.java  |  15 +-
 .../sql/action/SqlClearCursorRequest.java     |   2 -
 .../xpack/sql/action/SqlQueryRequest.java     |   2 +-
 .../sql/action/SqlQueryRequestBuilder.java    |   5 -
 .../xpack/sql/action/SqlTranslateRequest.java |  12 +-
 .../sql/action/SqlQueryRequestTests.java      |   3 -
 .../sql/action/SqlRequestParsersTests.java    |  20 +-
 .../org/elasticsearch/xpack/sql/cli/Cli.java  |   4 +-
 .../xpack/sql/cli/command/CliSession.java     |  15 +-
 .../sql/cli/command/PrintLogoCommand.java     |   6 +-
 .../xpack/sql/cli/CliSessionTests.java        |  38 +-
 .../xpack/sql/cli/VersionTests.java           |  10 +-
 .../sql/cli/command/BuiltinCommandTests.java  |   4 +-
 .../sql/client/ConnectionConfiguration.java   |   2 +
 .../xpack/sql/client/HttpClient.java          |   2 +-
 .../{ClientVersion.java => Version.java}      |  76 +-
 .../sql/client/HttpClientRequestTests.java    |   2 +-
 .../xpack/sql/client/VersionTests.java        |  42 +-
 .../xpack/sql/proto/AbstractSqlRequest.java   |   4 -
 .../elasticsearch/xpack/sql/proto/Mode.java   |   4 -
 .../xpack/sql/proto/RequestInfo.java          |  23 +-
 .../sql/proto/SqlClearCursorRequest.java      |   3 -
 .../xpack/sql/proto/SqlQueryRequest.java      |   3 -
 .../xpack/sql/proto/SqlVersion.java           | 144 ---
 .../xpack/sql/proto/SqlVersionTests.java      |  63 --
 .../xpack/sql/parser/ExpressionBuilder.java   |   6 +-
 .../xpack/sql/plugin/RestSqlQueryAction.java  |   2 +-
 .../xpack/sql/type/SqlDataTypeConverter.java  |   3 +-
 .../xpack/sql/type/SqlDataTypes.java          |  11 -
 .../xpack/sql/action/SqlActionIT.java         |  17 +-
 .../analysis/index/IndexResolverTests.java    |  51 +-
 .../xpack/sql/optimizer/OptimizerTests.java   |   3 +-
 .../xpack/sql/parser/ExpressionTests.java     |  10 -
 .../logical/command/sys/SysTypesTests.java    |   6 +-
 .../sql/planner/QueryTranslatorTests.java     |  17 -
 .../sql/type/SqlDataTypeConverterTests.java   |   7 -
 .../test/rest/CatIndicesWithSecurityIT.java   | 127 ---
 .../xpack/test/rest/XPackRestIT.java          |   9 +-
 .../api/async_search.delete.json              |  24 -
 .../rest-api-spec/api/async_search.get.json   |  40 -
 .../api/async_search.submit.json              | 225 -----
 .../api/ml.estimate_model_memory.json         |  23 -
 .../test/async_search/10_basic.yml            | 143 ---
 .../test/ml/data_frame_analytics_crud.yml     |  86 +-
 .../test/ml/estimate_model_memory.yml         | 172 ----
 .../rest-api-spec/test/ml/index_layout.yml    |  10 +-
 .../rest-api-spec/test/ml/inference_crud.yml  |   2 +-
 .../test/ml/inference_processor.yml           | 114 ---
 .../test/ml/inference_stats_crud.yml          |   4 +-
 .../rest-api-spec/test/ml/jobs_crud.yml       |   8 +-
 .../test/ml/start_data_frame_analytics.yml    |  21 +-
 .../test/wildcard/10_wildcard_basic.yml       | 218 -----
 x-pack/plugin/src/test/resources/roles.yml    |   7 -
 .../watcher/WatcherYamlSuiteTestCase.java     |  10 +-
 x-pack/plugin/wildcard/build.gradle           |  18 -
 .../xpack/wildcard/Wildcard.java              |  31 -
 .../mapper/AutomatonQueryOnBinaryDv.java      | 104 ---
 .../wildcard/mapper/WildcardFieldMapper.java  | 575 ------------
 .../mapper/WildcardFieldMapperTests.java      | 331 -------
 .../mapper/WildcardFieldTypeTests.java        |  19 -
 ...nfigIndexMappingsFullClusterRestartIT.java |  11 +-
 .../90_ml_data_frame_analytics_crud.yml       |  30 -
 .../90_ml_data_frame_analytics_crud.yml       |  36 -
 .../90_ml_data_frame_analytics_crud.yml       |  31 -
 684 files changed, 6014 insertions(+), 20435 deletions(-)
 delete mode 100644 docs/reference/images/analysis/token-graph-basic.svg
 delete mode 100644 docs/reference/images/analysis/token-graph-wd.svg
 delete mode 100644 docs/reference/images/analysis/token-graph-wdg.svg
 delete mode 100644 docs/reference/mapping/types/wildcard.asciidoc
 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
 delete mode 100644 server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/src/main/java/org/apache/lucene/queries/XIntervals.java
 delete mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
 delete mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
 delete mode 100644 server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
 delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java
 delete mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
 delete mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java
 delete mode 100644 x-pack/plugin/async-search/build.gradle
 delete mode 100644 x-pack/plugin/async-search/qa/build.gradle
 delete mode 100644 x-pack/plugin/async-search/qa/security/build.gradle
 delete mode 100644 x-pack/plugin/async-search/qa/security/roles.yml
 delete mode 100644 x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java
 delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java
 delete mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json
 delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java
 delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
 delete mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
 delete mode 100644 x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
 delete mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java
 delete mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
 delete mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
 create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java
 delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
 delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java
 delete mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java
 delete mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
 delete mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
 delete mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
 delete mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
 delete mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
 delete mode 100644 x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
 delete mode 100644 x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java
 delete mode 100644 x-pack/plugin/security/qa/security-trial/build.gradle
 delete mode 100644 x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
 delete mode 100644 x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
 delete mode 100644 x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml
 delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java
 delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java
 delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java
 delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java
 delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
 delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java
 delete mode 100644 x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
 rename x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/{ClientVersion.java => Version.java} (57%)
 delete mode 100644 x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java
 delete mode 100644 x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java
 delete mode 100644 x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
 delete mode 100644 x-pack/plugin/src/test/resources/roles.yml
 delete mode 100644 x-pack/plugin/wildcard/build.gradle
 delete mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
 delete mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
 delete mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
 delete mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
 delete mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java

diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
index b8c543cea0b08..4a0065b27afd4 100644
--- a/.ci/matrix-runtime-javas.yml
+++ b/.ci/matrix-runtime-javas.yml
@@ -7,6 +7,7 @@
 
 ES_RUNTIME_JAVA:
   - java11
+  - openjdk13
   - openjdk14
   - openjdk15
   - zulu11
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index eeea7b412cbf8..1f1a4720a2dd8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -171,13 +171,13 @@ IntelliJ or Eclipse like describe above to use
 Elasticsearch typically uses singular nouns rather than plurals in URLs.
 For example:
 
-    /_ingest/pipeline
-    /_ingest/pipeline/{id}
+    /_ingest/pipline
+    /_ingest/pipline/{id}
 
 but not:
 
-    /_ingest/pipelines
-    /_ingest/pipelines/{id}
+    /_ingest/piplines
+    /_ingest/piplines/{id}
 
 You may find counterexamples, but new endpoints should use the singular
 form.
diff --git a/build.gradle b/build.gradle
index 78322b938e6f4..5cee1f108cd69 100644
--- a/build.gradle
+++ b/build.gradle
@@ -118,8 +118,7 @@ subprojects {
       ':qa:os',
       ':qa:wildfly',
       ':x-pack:plugin:autoscaling',
-      ':x-pack:plugin:enrich',
-      ':x-pack:plugin:logstash'
+      ':x-pack:plugin:enrich'
     ]
 
     if (projectPathsToFormat.contains(project.path)) {
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
index 0982fe0267cc5..63aedd6566d89 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -174,21 +174,24 @@ private void setupRootDownload(Project rootProject, ElasticsearchDistribution di
     }
 
     private static void addIvyRepo(Project project, String name, String url, String group) {
-        IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
-            repo.setName(name);
-            repo.setUrl(url);
-            repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+        project.getRepositories().ivy(ivyRepo -> {
+            ivyRepo.setName(name);
+            ivyRepo.setUrl(url);
+            ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
             // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
-            repo.credentials(HttpHeaderCredentials.class, creds -> {
+            ivyRepo.credentials(HttpHeaderCredentials.class, creds -> {
                 creds.setName("X-Elastic-No-KPI");
                 creds.setValue("1");
             });
-            repo.getAuthentication().create("header", HttpHeaderAuthentication.class);
-            repo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
+            ivyRepo.getAuthentication().create("header", HttpHeaderAuthentication.class);
+            ivyRepo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
+            ivyRepo.content(content -> content.includeGroup(group));
         });
-        project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
-            exclusiveContentRepository.filter(config -> config.includeGroup(group));
-            exclusiveContentRepository.forRepositories(ivyRepo);
+        project.getRepositories().all(repo -> {
+            if (repo.getName().equals(name) == false) {
+                // all other repos should ignore the special group name
+                repo.content(content -> content.excludeGroup(group));
+            }
         });
     }
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
index b63d8f65ea04f..9c48fd8ac1b19 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
@@ -79,6 +79,16 @@ public void apply(Project project) {
                 setupRootJdkDownload(project.getRootProject(), jdk);
             }
         });
+
+        // all other repos should ignore the special jdk artifacts
+        project.getRootProject().getRepositories().all(repo -> {
+            if (repo.getName().startsWith(REPO_NAME_PREFIX) == false) {
+                repo.content(content -> {
+                    content.excludeGroup("adoptopenjdk");
+                    content.excludeGroup("openjdk");
+                });
+            }
+        });
     }
 
     @SuppressWarnings("unchecked")
@@ -135,16 +145,13 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
             }
 
             // Define the repository if we haven't already
-            if (repositories.findByName(repoName) == null) {
-                IvyArtifactRepository ivyRepo = repositories.ivy(repo -> {
-                    repo.setName(repoName);
-                    repo.setUrl(repoUrl);
-                    repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
-                    repo.patternLayout(layout -> layout.artifact(artifactPattern));
-                });
-                repositories.exclusiveContent(exclusiveContentRepository -> {
-                    exclusiveContentRepository.filter(config -> config.includeGroup(groupName(jdk)));
-                    exclusiveContentRepository.forRepositories(ivyRepo);
+            if (rootProject.getRepositories().findByName(repoName) == null) {
+                repositories.ivy(ivyRepo -> {
+                    ivyRepo.setName(repoName);
+                    ivyRepo.setUrl(repoUrl);
+                    ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                    ivyRepo.patternLayout(layout -> layout.artifact(artifactPattern));
+                    ivyRepo.content(content -> content.includeGroup(jdk.getVendor()));
                 });
             }
 
@@ -254,11 +261,7 @@ private static String dependencyNotation(Jdk jdk) {
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 
-        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
-    }
-
-    private static String groupName(Jdk jdk) {
-        return jdk.getVendor() + "_" + jdk.getMajor();
+        return jdk.getVendor() + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
     }
 
     private static String configName(String... parts) {
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index dea67de1657d1..9d654e0e6682b 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.5.0-snapshot-7f057455901
+lucene            = 8.5.0-snapshot-c4475920b08
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.2+8
@@ -11,7 +11,7 @@ jts               = 1.15.0
 # you should also inspect that version to see if it can be advanced along with
 # the com.maxmind.geoip2:geoip2 dependency
 jackson           = 2.8.11
-jacksondatabind   = 2.8.11.6
+jacksondatabind   = 2.8.11.4
 snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java
index 12f3a64e63cb2..2509dcb767449 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -86,9 +88,9 @@ public static class TopMetrics implements ToXContent {
         private static final ParseField METRICS_FIELD = new ParseField("metrics");
 
         private final List<Object> sort;
-        private final Map<String, Object> metrics;
+        private final Map<String, Double> metrics;
 
-        private TopMetrics(List<Object> sort, Map<String, Object> metrics) {
+        private TopMetrics(List<Object> sort, Map<String, Double> metrics) {
             this.sort = sort;
             this.metrics = metrics;
         }
@@ -103,7 +105,7 @@ public List<Object> getSort() {
         /**
          * The top metric values returned by the aggregation.
          */
-        public Map<String, Object> getMetrics() {
+        public Map<String, Double> getMetrics() {
             return metrics;
         }
 
@@ -112,13 +114,13 @@ public Map<String, Object> getMetrics() {
                     @SuppressWarnings("unchecked")
                     List<Object> sort = (List<Object>) args[0];
                     @SuppressWarnings("unchecked")
-                    Map<String, Object> metrics = (Map<String, Object>) args[1];
+                    Map<String, Double> metrics = (Map<String, Double>) args[1];
                     return new TopMetrics(sort, metrics);
                 });
         static {
             PARSER.declareFieldArray(constructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p),
                     SORT_FIELD, ObjectParser.ValueType.VALUE_ARRAY);
-            PARSER.declareObject(constructorArg(), (p, c) -> p.map(), METRICS_FIELD);
+            PARSER.declareObject(constructorArg(), (p, c) -> p.map(HashMap::new, XContentParser::doubleValue), METRICS_FIELD);
         }
 
         public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
index 79758ec709c63..0fc4f240eb8ef 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
@@ -23,9 +23,9 @@
 
 public class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
     static final String NAME = "proxy";
-    static final String PROXY_ADDRESS = "proxy_address";
-    static final String NUM_PROXY_SOCKETS_CONNECTED = "num_proxy_sockets_connected";
-    static final String MAX_PROXY_SOCKET_CONNECTIONS = "max_proxy_socket_connections";
+    static final String ADDRESS = "address";
+    static final String NUM_SOCKETS_CONNECTED = "num_sockets_connected";
+    static final String MAX_SOCKET_CONNECTIONS = "max_socket_connections";
     private final String address;
     private final int maxSocketConnections;
     private final int numSocketsConnected;
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
index 474b991cd434a..2bf99c61085c4 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
@@ -66,9 +66,9 @@ public final class RemoteConnectionInfo {
         PARSER.declareString(constructorArg(), new ParseField(INITIAL_CONNECT_TIMEOUT));
         PARSER.declareBoolean(constructorArg(), new ParseField(SKIP_UNAVAILABLE));
 
-        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.PROXY_ADDRESS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_PROXY_SOCKET_CONNECTIONS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_PROXY_SOCKETS_CONNECTED));
+        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.ADDRESS));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_SOCKET_CONNECTIONS));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_SOCKETS_CONNECTED));
 
         PARSER.declareStringArray(optionalConstructorArg(), new ParseField(SniffModeInfo.SEEDS));
         PARSER.declareInt(optionalConstructorArg(), new ParseField(SniffModeInfo.MAX_CONNECTIONS_PER_CLUSTER));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
index 48c75e6e34e6d..02861adc73845 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
@@ -22,12 +22,10 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Locale;
 import java.util.Objects;
 
 public class Classification implements DataFrameAnalysis {
@@ -46,12 +44,11 @@ public static Builder builder(String dependentVariable) {
     static final ParseField LAMBDA = new ParseField("lambda");
     static final ParseField GAMMA = new ParseField("gamma");
     static final ParseField ETA = new ParseField("eta");
-    static final ParseField MAX_TREES = new ParseField("max_trees");
+    static final ParseField MAXIMUM_NUMBER_TREES = new ParseField("maximum_number_trees");
     static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
     static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
     static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
-    static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
     static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
     static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
@@ -70,57 +67,47 @@ public static Builder builder(String dependentVariable) {
                 (String) a[7],
                 (Double) a[8],
                 (Integer) a[9],
-                (Long) a[10],
-                (ClassAssignmentObjective) a[11]));
+                (Long) a[10]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DEPENDENT_VARIABLE);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), LAMBDA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), GAMMA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), ETA);
-        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_TREES);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_NUMBER_TREES);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), FEATURE_BAG_FRACTION);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_FEATURE_IMPORTANCE_VALUES);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PREDICTION_FIELD_NAME);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), TRAINING_PERCENT);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_CLASSES);
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), RANDOMIZE_SEED);
-        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
-            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                return ClassAssignmentObjective.fromString(p.text());
-            }
-            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-        }, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
     }
 
     private final String dependentVariable;
     private final Double lambda;
     private final Double gamma;
     private final Double eta;
-    private final Integer maxTrees;
+    private final Integer maximumNumberTrees;
     private final Double featureBagFraction;
     private final Integer numTopFeatureImportanceValues;
     private final String predictionFieldName;
     private final Double trainingPercent;
-    private final ClassAssignmentObjective classAssignmentObjective;
     private final Integer numTopClasses;
     private final Long randomizeSeed;
 
     private Classification(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
-                           @Nullable Integer maxTrees, @Nullable Double featureBagFraction,
+                           @Nullable Integer maximumNumberTrees, @Nullable Double featureBagFraction,
                            @Nullable Integer numTopFeatureImportanceValues, @Nullable String predictionFieldName,
-                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed,
-                           @Nullable ClassAssignmentObjective classAssignmentObjective) {
+                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
         this.eta = eta;
-        this.maxTrees = maxTrees;
+        this.maximumNumberTrees = maximumNumberTrees;
         this.featureBagFraction = featureBagFraction;
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
         this.predictionFieldName = predictionFieldName;
         this.trainingPercent = trainingPercent;
-        this.classAssignmentObjective = classAssignmentObjective;
         this.numTopClasses = numTopClasses;
         this.randomizeSeed = randomizeSeed;
     }
@@ -146,8 +133,8 @@ public Double getEta() {
         return eta;
     }
 
-    public Integer getMaxTrees() {
-        return maxTrees;
+    public Integer getMaximumNumberTrees() {
+        return maximumNumberTrees;
     }
 
     public Double getFeatureBagFraction() {
@@ -170,10 +157,6 @@ public Long getRandomizeSeed() {
         return randomizeSeed;
     }
 
-    public ClassAssignmentObjective getClassAssignmentObjective() {
-        return classAssignmentObjective;
-    }
-
     public Integer getNumTopClasses() {
         return numTopClasses;
     }
@@ -191,8 +174,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (eta != null) {
             builder.field(ETA.getPreferredName(), eta);
         }
-        if (maxTrees != null) {
-            builder.field(MAX_TREES.getPreferredName(), maxTrees);
+        if (maximumNumberTrees != null) {
+            builder.field(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
         }
         if (featureBagFraction != null) {
             builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -209,9 +192,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (randomizeSeed != null) {
             builder.field(RANDOMIZE_SEED.getPreferredName(), randomizeSeed);
         }
-        if (classAssignmentObjective != null) {
-            builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
-        }
         if (numTopClasses != null) {
             builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         }
@@ -221,8 +201,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues,
-            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses, classAssignmentObjective);
+        return Objects.hash(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues,
+            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses);
     }
 
     @Override
@@ -234,14 +214,13 @@ public boolean equals(Object o) {
             && Objects.equals(lambda, that.lambda)
             && Objects.equals(gamma, that.gamma)
             && Objects.equals(eta, that.eta)
-            && Objects.equals(maxTrees, that.maxTrees)
+            && Objects.equals(maximumNumberTrees, that.maximumNumberTrees)
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(numTopFeatureImportanceValues, that.numTopFeatureImportanceValues)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
             && Objects.equals(trainingPercent, that.trainingPercent)
             && Objects.equals(randomizeSeed, that.randomizeSeed)
-            && Objects.equals(numTopClasses, that.numTopClasses)
-            && Objects.equals(classAssignmentObjective, that.classAssignmentObjective);
+            && Objects.equals(numTopClasses, that.numTopClasses);
     }
 
     @Override
@@ -249,32 +228,18 @@ public String toString() {
         return Strings.toString(this);
     }
 
-    public enum ClassAssignmentObjective {
-        MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
-
-        public static ClassAssignmentObjective fromString(String value) {
-            return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
-        }
-
-        @Override
-        public String toString() {
-            return name().toLowerCase(Locale.ROOT);
-        }
-    }
-
     public static class Builder {
         private String dependentVariable;
         private Double lambda;
         private Double gamma;
         private Double eta;
-        private Integer maxTrees;
+        private Integer maximumNumberTrees;
         private Double featureBagFraction;
         private Integer numTopFeatureImportanceValues;
         private String predictionFieldName;
         private Double trainingPercent;
         private Integer numTopClasses;
         private Long randomizeSeed;
-        private ClassAssignmentObjective classAssignmentObjective;
 
         private Builder(String dependentVariable) {
             this.dependentVariable = Objects.requireNonNull(dependentVariable);
@@ -295,8 +260,8 @@ public Builder setEta(Double eta) {
             return this;
         }
 
-        public Builder setMaxTrees(Integer maxTrees) {
-            this.maxTrees = maxTrees;
+        public Builder setMaximumNumberTrees(Integer maximumNumberTrees) {
+            this.maximumNumberTrees = maximumNumberTrees;
             return this;
         }
 
@@ -330,15 +295,9 @@ public Builder setNumTopClasses(Integer numTopClasses) {
             return this;
         }
 
-        public Builder setClassAssignmentObjective(ClassAssignmentObjective classAssignmentObjective) {
-            this.classAssignmentObjective = classAssignmentObjective;
-            return this;
-        }
-
         public Classification build() {
-            return new Classification(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction,
-                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed,
-                classAssignmentObjective);
+            return new Classification(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction,
+                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed);
         }
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java
index 7eb1af3405486..d7e374a2563a1 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java
@@ -44,7 +44,7 @@ public static Builder builder(String dependentVariable) {
     static final ParseField LAMBDA = new ParseField("lambda");
     static final ParseField GAMMA = new ParseField("gamma");
     static final ParseField ETA = new ParseField("eta");
-    static final ParseField MAX_TREES = new ParseField("max_trees");
+    static final ParseField MAXIMUM_NUMBER_TREES = new ParseField("maximum_number_trees");
     static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
     static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
@@ -72,7 +72,7 @@ public static Builder builder(String dependentVariable) {
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), LAMBDA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), GAMMA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), ETA);
-        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_TREES);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_NUMBER_TREES);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), FEATURE_BAG_FRACTION);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_FEATURE_IMPORTANCE_VALUES);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PREDICTION_FIELD_NAME);
@@ -84,7 +84,7 @@ public static Builder builder(String dependentVariable) {
     private final Double lambda;
     private final Double gamma;
     private final Double eta;
-    private final Integer maxTrees;
+    private final Integer maximumNumberTrees;
     private final Double featureBagFraction;
     private final Integer numTopFeatureImportanceValues;
     private final String predictionFieldName;
@@ -92,14 +92,14 @@ public static Builder builder(String dependentVariable) {
     private final Long randomizeSeed;
 
     private Regression(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
-                       @Nullable Integer maxTrees, @Nullable Double featureBagFraction,
+                       @Nullable Integer maximumNumberTrees, @Nullable Double featureBagFraction,
                        @Nullable Integer numTopFeatureImportanceValues, @Nullable String predictionFieldName,
                        @Nullable Double trainingPercent, @Nullable Long randomizeSeed) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
         this.eta = eta;
-        this.maxTrees = maxTrees;
+        this.maximumNumberTrees = maximumNumberTrees;
         this.featureBagFraction = featureBagFraction;
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
         this.predictionFieldName = predictionFieldName;
@@ -128,8 +128,8 @@ public Double getEta() {
         return eta;
     }
 
-    public Integer getMaxTrees() {
-        return maxTrees;
+    public Integer getMaximumNumberTrees() {
+        return maximumNumberTrees;
     }
 
     public Double getFeatureBagFraction() {
@@ -165,8 +165,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (eta != null) {
             builder.field(ETA.getPreferredName(), eta);
         }
-        if (maxTrees != null) {
-            builder.field(MAX_TREES.getPreferredName(), maxTrees);
+        if (maximumNumberTrees != null) {
+            builder.field(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
         }
         if (featureBagFraction != null) {
             builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -189,7 +189,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues,
+        return Objects.hash(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues,
             predictionFieldName, trainingPercent, randomizeSeed);
     }
 
@@ -202,7 +202,7 @@ public boolean equals(Object o) {
             && Objects.equals(lambda, that.lambda)
             && Objects.equals(gamma, that.gamma)
             && Objects.equals(eta, that.eta)
-            && Objects.equals(maxTrees, that.maxTrees)
+            && Objects.equals(maximumNumberTrees, that.maximumNumberTrees)
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(numTopFeatureImportanceValues, that.numTopFeatureImportanceValues)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
@@ -220,7 +220,7 @@ public static class Builder {
         private Double lambda;
         private Double gamma;
         private Double eta;
-        private Integer maxTrees;
+        private Integer maximumNumberTrees;
         private Double featureBagFraction;
         private Integer numTopFeatureImportanceValues;
         private String predictionFieldName;
@@ -246,8 +246,8 @@ public Builder setEta(Double eta) {
             return this;
         }
 
-        public Builder setMaxTrees(Integer maxTrees) {
-            this.maxTrees = maxTrees;
+        public Builder setMaximumNumberTrees(Integer maximumNumberTrees) {
+            this.maximumNumberTrees = maximumNumberTrees;
             return this;
         }
 
@@ -277,7 +277,7 @@ public Builder setRandomizeSeed(Long randomizeSeed) {
         }
 
         public Regression build() {
-            return new Regression(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction,
+            return new Regression(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction,
                 numTopFeatureImportanceValues, predictionFieldName, trainingPercent, randomizeSeed);
         }
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java
index d8749c0a6cdc9..9d2b323cf4880 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java
@@ -53,7 +53,6 @@ public class TrainedModelConfig implements ToXContentObject {
     public static final ParseField ESTIMATED_HEAP_MEMORY_USAGE_BYTES = new ParseField("estimated_heap_memory_usage_bytes");
     public static final ParseField ESTIMATED_OPERATIONS = new ParseField("estimated_operations");
     public static final ParseField LICENSE_LEVEL = new ParseField("license_level");
-    public static final ParseField DEFAULT_FIELD_MAP = new ParseField("default_field_map");
 
     public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(NAME,
             true,
@@ -77,7 +76,6 @@ public class TrainedModelConfig implements ToXContentObject {
         PARSER.declareLong(TrainedModelConfig.Builder::setEstimatedHeapMemory, ESTIMATED_HEAP_MEMORY_USAGE_BYTES);
         PARSER.declareLong(TrainedModelConfig.Builder::setEstimatedOperations, ESTIMATED_OPERATIONS);
         PARSER.declareString(TrainedModelConfig.Builder::setLicenseLevel, LICENSE_LEVEL);
-        PARSER.declareObject(TrainedModelConfig.Builder::setDefaultFieldMap, (p, c) -> p.mapStrings(), DEFAULT_FIELD_MAP);
     }
 
     public static TrainedModelConfig fromXContent(XContentParser parser) throws IOException {
@@ -97,7 +95,6 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
     private final Long estimatedHeapMemory;
     private final Long estimatedOperations;
     private final String licenseLevel;
-    private final Map<String, String> defaultFieldMap;
 
     TrainedModelConfig(String modelId,
                        String createdBy,
@@ -111,8 +108,7 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
                        TrainedModelInput input,
                        Long estimatedHeapMemory,
                        Long estimatedOperations,
-                       String licenseLevel,
-                       Map<String, String> defaultFieldMap) {
+                       String licenseLevel) {
         this.modelId = modelId;
         this.createdBy = createdBy;
         this.version = version;
@@ -126,7 +122,6 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
         this.estimatedHeapMemory = estimatedHeapMemory;
         this.estimatedOperations = estimatedOperations;
         this.licenseLevel = licenseLevel;
-        this.defaultFieldMap = defaultFieldMap == null ? null : Collections.unmodifiableMap(defaultFieldMap);
     }
 
     public String getModelId() {
@@ -185,10 +180,6 @@ public String getLicenseLevel() {
         return licenseLevel;
     }
 
-    public Map<String, String> getDefaultFieldMap() {
-        return defaultFieldMap;
-    }
-
     public static Builder builder() {
         return new Builder();
     }
@@ -235,9 +226,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (licenseLevel != null) {
             builder.field(LICENSE_LEVEL.getPreferredName(), licenseLevel);
         }
-        if (defaultFieldMap != null) {
-            builder.field(DEFAULT_FIELD_MAP.getPreferredName(), defaultFieldMap);
-        }
         builder.endObject();
         return builder;
     }
@@ -264,7 +252,6 @@ public boolean equals(Object o) {
             Objects.equals(estimatedHeapMemory, that.estimatedHeapMemory) &&
             Objects.equals(estimatedOperations, that.estimatedOperations) &&
             Objects.equals(licenseLevel, that.licenseLevel) &&
-            Objects.equals(defaultFieldMap, that.defaultFieldMap) &&
             Objects.equals(metadata, that.metadata);
     }
 
@@ -282,8 +269,7 @@ public int hashCode() {
             estimatedOperations,
             metadata,
             licenseLevel,
-            input,
-            defaultFieldMap);
+            input);
     }
 
 
@@ -302,7 +288,6 @@ public static class Builder {
         private Long estimatedHeapMemory;
         private Long estimatedOperations;
         private String licenseLevel;
-        private Map<String, String> defaultFieldMap;
 
         public Builder setModelId(String modelId) {
             this.modelId = modelId;
@@ -382,11 +367,6 @@ private Builder setLicenseLevel(String licenseLevel) {
             return this;
         }
 
-        public Builder setDefaultFieldMap(Map<String, String> defaultFieldMap) {
-            this.defaultFieldMap = defaultFieldMap;
-            return this;
-        }
-
         public TrainedModelConfig build() {
             return new TrainedModelConfig(
                 modelId,
@@ -401,8 +381,7 @@ public TrainedModelConfig build() {
                 input,
                 estimatedHeapMemory,
                 estimatedOperations,
-                licenseLevel,
-                defaultFieldMap);
+                licenseLevel);
         }
     }
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
index 1f6eed5b3fac1..e01d08d019fdc 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
@@ -43,8 +43,9 @@ public class TrainedModelDefinition implements ToXContentObject {
             true,
             TrainedModelDefinition.Builder::new);
     static {
-        PARSER.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
+        PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
             (p, c, n) -> p.namedObject(TrainedModel.class, n, null),
+            (modelDocBuilder) -> { /* Noop does not matter client side*/ },
             TRAINED_MODEL);
         PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
             (p, c, n) -> p.namedObject(PreProcessor.class, n, null),
@@ -123,6 +124,11 @@ public Builder setTrainedModel(TrainedModel trainedModel) {
             return this;
         }
 
+        private Builder setTrainedModel(List<TrainedModel> trainedModel) {
+            assert trainedModel.size() == 1;
+            return setTrainedModel(trainedModel.get(0));
+        }
+
         public TrainedModelDefinition build() {
             return new TrainedModelDefinition(this.trainedModel, this.preProcessors);
         }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
index 4f16422cc7bd7..45d8afb32e6b9 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -56,8 +56,9 @@ public class Ensemble implements TrainedModel {
                     p.namedObject(TrainedModel.class, n, null),
             (ensembleBuilder) -> { /* Noop does not matter client side */ },
             TRAINED_MODELS);
-        PARSER.declareNamedObject(Ensemble.Builder::setOutputAggregator,
+        PARSER.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
             (p, c, n) -> p.namedObject(OutputAggregator.class, n, null),
+            (ensembleBuilder) -> { /* Noop does not matter client side */ },
             AGGREGATE_OUTPUT);
         PARSER.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
         PARSER.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -193,6 +194,9 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
             return this;
         }
 
+        private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
+            this.setOutputAggregator(outputAggregators.get(0));
+        }
 
         private void setTargetType(String targetType) {
             this.targetType = TargetType.fromString(targetType);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java
index 92ed0ea83073c..10e75d88307b6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java
@@ -21,12 +21,6 @@
 
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
-import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
-import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.eql.EqlSearchRequest;
 import org.elasticsearch.client.eql.EqlSearchResponse;
 import org.elasticsearch.client.eql.EqlStatsRequest;
@@ -34,132 +28,51 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.time.format.DateTimeFormatter;
-import java.util.Locale;
-import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
 public class EqlIT extends ESRestHighLevelClientTestCase {
 
-    private static final String INDEX_NAME = "index";
-    private static final int RECORD_COUNT = 40;
-    private static final int DIVIDER = 4;
-
     @Before
-    public void setup() throws Exception {
+    public void setupRemoteClusterConfig() throws Exception {
         setupRemoteClusterConfig("local_cluster");
-        setupData();
     }
 
-    private void setupData() throws IOException {
-        final BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
-        for (int i = 0; i < RECORD_COUNT; i++) {
-            final IndexRequest indexRequest = new IndexRequest(INDEX_NAME);
-            indexRequest.source(jsonBuilder()
-                    .startObject()
-                    .field("event_subtype_full", "already_running")
-                    .startObject("event")
-                    .field("category", "process")
-                    .endObject()
-                    .field("event_type", "foo")
-                    .field("event_type_full", "process_event")
-                    .field("opcode", ((i % DIVIDER) == 0) ? 1 : 0)
-                    .field("pid", ((i % DIVIDER) == 0) ? 100 : 0)
-                    .field("process_name", "System Idle Process")
-                    .field("serial_event_id", i + 1)
-                    .field("subtype", "create")
-                    .field("@timestamp", String.format(Locale.ROOT, "2018-01-01T00:00:%02dZ", i))
-                    .field("unique_pid", ((i % DIVIDER) == 0) ? 101 : 0)
-                    .endObject());
-            bulkRequest.add(indexRequest);
-        }
-        BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT);
-        assertEquals(RestStatus.OK, bulkResponse.status());
-        assertFalse(bulkResponse.hasFailures());
-
-        RefreshResponse refreshResponse = highLevelClient().indices().refresh(new RefreshRequest(INDEX_NAME), RequestOptions.DEFAULT);
-        assertEquals(0, refreshResponse.getFailedShards());
-    }
+    public void testBasicSearch() throws Exception {
+        Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
+        doc1.setJsonEntity("{\"event_subtype_full\": \"already_running\", " +
+                "\"event\": {" +
+                    "\"category\": \"process\"" +
+                "}," +
+                "\"event_type_full\": \"process_event\", " +
+                "\"opcode\": 3," +
+                "\"pid\": 0," +
+                "\"process_name\": \"System Idle Process\"," +
+                "\"serial_event_id\": 1," +
+                "\"subtype\": \"create\"," +
+                "\"@timestamp\": 116444736000000000," +
+                "\"unique_pid\": 1}");
+        client().performRequest(doc1);
+        client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh"));
 
-    private void assertResponse(EqlSearchResponse response, int count) {
+        EqlClient eql = highLevelClient().eql();
+        EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
+        EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
         assertNotNull(response);
         assertFalse(response.isTimeout());
         assertNotNull(response.hits());
         assertNull(response.hits().sequences());
         assertNull(response.hits().counts());
         assertNotNull(response.hits().events());
-        assertThat(response.hits().events().size(), equalTo(count));
-    }
-
-    public void testBasicSearch() throws Exception {
-        EqlClient eql = highLevelClient().eql();
-        EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
-        assertResponse(execute(request, eql::search, eql::searchAsync), RECORD_COUNT);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testSimpleConditionSearch() throws Exception {
-        EqlClient eql = highLevelClient().eql();
-
-        // test simple conditional
-        EqlSearchRequest request = new EqlSearchRequest("index", "foo where pid > 0");
-
-        // test with non-default event.category mapping
-        request.eventCategoryField("event_type");
-
-        EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
-        assertResponse(response, RECORD_COUNT / DIVIDER);
-
-        // test the content of the hits
-        for (SearchHit hit : response.hits().events()) {
-            final Map<String, Object> source = hit.getSourceAsMap();
-
-            final Map<String, Object> event = (Map<String, Object>) source.get("event");
-            assertThat(event.get("category"), equalTo("process"));
-            assertThat(source.get("event_type"), equalTo("foo"));
-            assertThat(source.get("event_type_full"), equalTo("process_event"));
-            assertThat(source.get("opcode"), equalTo(1));
-            assertThat(source.get("pid"), equalTo(100));
-            assertThat(source.get("process_name"), equalTo("System Idle Process"));
-            assertThat((int) source.get("serial_event_id"), greaterThan(0));
-            assertThat(source.get("unique_pid"), equalTo(101));
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testEqualsInFilterConditionSearch() throws Exception {
-        EqlClient eql = highLevelClient().eql();
-
-        EqlSearchRequest request = new EqlSearchRequest("index",
-                "process where event_type_full = \"process_event\" and serial_event_id in (1,3,5)");
-
-        EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
-        assertResponse(response, 3);
-
-        // test the content of the hits
-        for (SearchHit hit : response.hits().events()) {
-            final Map<String, Object> source = hit.getSourceAsMap();
-
-            final Map<String, Object> event = (Map<String, Object>) source.get("event");
-            assertThat(event.get("category"), equalTo("process"));
-            assertThat(source.get("serial_event_id"), anyOf(equalTo(1), equalTo(3), equalTo(5)));
-        }
+        assertThat(response.hits().events().size(), equalTo(1));
     }
 
     public void testLargeMapping() throws Exception {
-        final String index = "large_mapping_index";
-
-        Request doc1 = new Request(HttpPut.METHOD_NAME, "/" + index + "/_doc/1");
+        Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
         // use more exact fields (dates) than the default to verify that retrieval works and requesting doc values
         // would fail
         int PASS_DEFAULT_DOC_VALUES = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY) + 50;
@@ -181,7 +94,7 @@ public void testLargeMapping() throws Exception {
 
 
         EqlClient eql = highLevelClient().eql();
-        EqlSearchRequest request = new EqlSearchRequest(index, "process where true");
+        EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
         EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
         assertNotNull(response);
         assertNotNull(response.hits());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 17a9c0cd1cfca..3e9cc1f55ff38 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -1297,7 +1297,7 @@ public void testPutDataFrameAnalyticsConfig_GivenRegression() throws Exception {
                 .setLambda(1.0)
                 .setGamma(1.0)
                 .setEta(1.0)
-                .setMaxTrees(10)
+                .setMaximumNumberTrees(10)
                 .setFeatureBagFraction(0.5)
                 .setNumTopFeatureImportanceValues(3)
                 .build())
@@ -1336,13 +1336,11 @@ public void testPutDataFrameAnalyticsConfig_GivenClassification() throws Excepti
                 .setPredictionFieldName("my_dependent_variable_prediction")
                 .setTrainingPercent(80.0)
                 .setRandomizeSeed(42L)
-                .setClassAssignmentObjective(
-                    org.elasticsearch.client.ml.dataframe.Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY)
                 .setNumTopClasses(1)
                 .setLambda(1.0)
                 .setGamma(1.0)
                 .setEta(1.0)
-                .setMaxTrees(10)
+                .setMaximumNumberTrees(10)
                 .setFeatureBagFraction(0.5)
                 .setNumTopFeatureImportanceValues(3)
                 .build())
@@ -2267,7 +2265,7 @@ public void testGetTrainedModelsStats() throws Exception {
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_map\": {\n" +
+            "          \"field_mappings\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java
index 946c707754e04..03911a36884a5 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
@@ -62,8 +61,8 @@ public void testStringStats() throws IOException {
         assertThat(stats.getDistribution(), hasEntry(equalTo("t"), closeTo(.09, .005)));
     }
 
-    public void testTopMetricsDoubleMetric() throws IOException {
-        indexTopMetricsDoubleTestData();
+    public void testTopMetricsSizeOne() throws IOException {
+        indexTopMetricsData();
         SearchRequest search = new SearchRequest("test");
         search.source().aggregation(new TopMetricsAggregationBuilder(
                 "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v"));
@@ -75,34 +74,8 @@ public void testTopMetricsDoubleMetric() throws IOException {
         assertThat(metric.getMetrics(), equalTo(singletonMap("v", 3.0)));
     }
 
-    public void testTopMetricsLongMetric() throws IOException {
-        indexTopMetricsLongTestData();
-        SearchRequest search = new SearchRequest("test");
-        search.source().aggregation(new TopMetricsAggregationBuilder(
-                "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v"));
-        SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
-        ParsedTopMetrics top = response.getAggregations().get("test");
-        assertThat(top.getTopMetrics(), hasSize(1));
-        ParsedTopMetrics.TopMetrics metric = top.getTopMetrics().get(0);
-        assertThat(metric.getSort(), equalTo(singletonList(2)));
-        assertThat(metric.getMetrics(), equalTo(singletonMap("v", 3)));
-    }
-
-    public void testTopMetricsDateMetric() throws IOException {
-        indexTopMetricsDateTestData();
-        SearchRequest search = new SearchRequest("test");
-        search.source().aggregation(new TopMetricsAggregationBuilder(
-                "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v"));
-        SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
-        ParsedTopMetrics top = response.getAggregations().get("test");
-        assertThat(top.getTopMetrics(), hasSize(1));
-        ParsedTopMetrics.TopMetrics metric = top.getTopMetrics().get(0);
-        assertThat(metric.getSort(), equalTo(singletonList(2)));
-        assertThat(metric.getMetrics(), equalTo(singletonMap("v", "2020-01-02T01:01:00.000Z")));
-    }
-
     public void testTopMetricsManyMetrics() throws IOException {
-        indexTopMetricsDoubleTestData();
+        indexTopMetricsData();
         SearchRequest search = new SearchRequest("test");
         search.source().aggregation(new TopMetricsAggregationBuilder(
                 "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v", "m"));
@@ -116,7 +89,7 @@ public void testTopMetricsManyMetrics() throws IOException {
     }
 
     public void testTopMetricsSizeTwo() throws IOException {
-        indexTopMetricsDoubleTestData();
+        indexTopMetricsData();
         SearchRequest search = new SearchRequest("test");
         search.source().aggregation(new TopMetricsAggregationBuilder(
                 "test", new FieldSortBuilder("s").order(SortOrder.DESC), 2, "v"));
@@ -131,28 +104,10 @@ public void testTopMetricsSizeTwo() throws IOException {
         assertThat(metric.getMetrics(), equalTo(singletonMap("v", 2.0)));
     }
 
-    private void indexTopMetricsDoubleTestData() throws IOException {
+    private void indexTopMetricsData() throws IOException {
         BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
         bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", 2.0, "m", 12.0));
         bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", 3.0, "m", 13.0));
         highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
     }
-
-    private void indexTopMetricsLongTestData() throws IOException {
-        BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
-        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", 2));
-        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", 3));
-        highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
-    }
-
-    private void indexTopMetricsDateTestData() throws IOException {
-        CreateIndexRequest create = new CreateIndexRequest("test");
-        create.mapping("{\"properties\": {\"v\": {\"type\": \"date\"}}}", XContentType.JSON);
-        highLevelClient().indices().create(create, RequestOptions.DEFAULT);
-        BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
-        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", "2020-01-01T01:01:00Z"));
-        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", "2020-01-02T01:01:00Z"));
-        highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
-    }
-
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
index adf78daacc7d5..d1c3a5e657e0f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
@@ -139,7 +139,6 @@
 import org.elasticsearch.client.ml.datafeed.DatafeedStats;
 import org.elasticsearch.client.ml.datafeed.DatafeedUpdate;
 import org.elasticsearch.client.ml.datafeed.DelayedDataCheckConfig;
-import org.elasticsearch.client.ml.dataframe.Classification;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDest;
@@ -2970,18 +2969,17 @@ public void testPutDataFrameAnalytics() throws Exception {
             // end::put-data-frame-analytics-outlier-detection-customized
 
             // tag::put-data-frame-analytics-classification
-            DataFrameAnalysis classification = Classification.builder("my_dependent_variable") // <1>
+            DataFrameAnalysis classification = org.elasticsearch.client.ml.dataframe.Classification.builder("my_dependent_variable") // <1>
                 .setLambda(1.0) // <2>
                 .setGamma(5.5) // <3>
                 .setEta(5.5) // <4>
-                .setMaxTrees(50) // <5>
+                .setMaximumNumberTrees(50) // <5>
                 .setFeatureBagFraction(0.4) // <6>
                 .setNumTopFeatureImportanceValues(3) // <7>
                 .setPredictionFieldName("my_prediction_field_name") // <8>
                 .setTrainingPercent(50.0) // <9>
                 .setRandomizeSeed(1234L) // <10>
-                .setClassAssignmentObjective(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY) // <11>
-                .setNumTopClasses(1) // <12>
+                .setNumTopClasses(1) // <11>
                 .build();
             // end::put-data-frame-analytics-classification
 
@@ -2990,7 +2988,7 @@ public void testPutDataFrameAnalytics() throws Exception {
                 .setLambda(1.0) // <2>
                 .setGamma(5.5) // <3>
                 .setEta(5.5) // <4>
-                .setMaxTrees(50) // <5>
+                .setMaximumNumberTrees(50) // <5>
                 .setFeatureBagFraction(0.4) // <6>
                 .setNumTopFeatureImportanceValues(3) // <7>
                 .setPredictionFieldName("my_prediction_field_name") // <8>
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
index 0970222c513b9..79d78c888880f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
@@ -30,13 +30,12 @@ public static Classification randomClassification() {
             .setLambda(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setGamma(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setEta(randomBoolean() ? null : randomDoubleBetween(0.001, 1.0, true))
-            .setMaxTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
+            .setMaximumNumberTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
             .setFeatureBagFraction(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, false))
             .setNumTopFeatureImportanceValues(randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE))
             .setPredictionFieldName(randomBoolean() ? null : randomAlphaOfLength(10))
             .setTrainingPercent(randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true))
             .setRandomizeSeed(randomBoolean() ? null : randomLong())
-            .setClassAssignmentObjective(randomBoolean() ? null : randomFrom(Classification.ClassAssignmentObjective.values()))
             .setNumTopClasses(randomBoolean() ? null : randomIntBetween(0, 10))
             .build();
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java
index 1b81f1d0f71af..eedffb4740d78 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java
@@ -30,7 +30,7 @@ public static Regression randomRegression() {
             .setLambda(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setGamma(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setEta(randomBoolean() ? null : randomDoubleBetween(0.001, 1.0, true))
-            .setMaxTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
+            .setMaximumNumberTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
             .setFeatureBagFraction(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, false))
             .setNumTopFeatureImportanceValues(randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE))
             .setPredictionFieldName(randomBoolean() ? null : randomAlphaOfLength(10))
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java
index 1bcefc76434c5..43ab2e5993fde 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java
@@ -30,7 +30,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -53,11 +52,7 @@ public static TrainedModelConfig createTestTrainedModelConfig() {
             randomBoolean() ? null : TrainedModelInputTests.createRandomInput(),
             randomBoolean() ? null : randomNonNegativeLong(),
             randomBoolean() ? null : randomNonNegativeLong(),
-            randomBoolean() ? null : randomFrom("platinum", "basic"),
-            randomBoolean() ? null :
-                Stream.generate(() -> randomAlphaOfLength(10))
-                    .limit(randomIntBetween(1, 10))
-                    .collect(Collectors.toMap(Function.identity(), (k) -> randomAlphaOfLength(10))));
+            randomBoolean() ? null : randomFrom("platinum", "basic"));
     }
 
     @Override
diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml
index 08e39ff8dd750..245056382f304 100644
--- a/distribution/docker/docker-compose.yml
+++ b/distribution/docker/docker-compose.yml
@@ -1,5 +1,5 @@
 # Only used for testing the docker images
-version: '3.4'
+version: '3.7'
 services:
   elasticsearch-default-1:
     image: elasticsearch:test
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index f7d22f233b840..ac839e3b9cb95 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -29,7 +29,7 @@ ifeval::["{release-state}"=="unreleased"]
 :rest-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}-SNAPSHOT
 :rest-client-sniffer-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}-SNAPSHOT
 :rest-high-level-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}-SNAPSHOT
-:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}-SNAPSHOT
+:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}-SNAPSHOT
 :painless-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}-SNAPSHOT
 :parent-join-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}-SNAPSHOT
 :percolator-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}-SNAPSHOT
@@ -44,7 +44,7 @@ ifeval::["{release-state}"!="unreleased"]
 :rest-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}
 :rest-client-sniffer-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}
 :rest-high-level-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}
-:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}
+:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}
 :painless-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}
 :parent-join-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}
 :percolator-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}
diff --git a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
index cf88d65ae9314..4be2011340210 100644
--- a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
+++ b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
@@ -121,8 +121,7 @@ include-tagged::{doc-tests-file}[{api}-classification]
 <8> The name of the prediction field in the results object.
 <9> The percentage of training-eligible rows to be used in training. Defaults to 100%.
 <10> The seed to be used by the random generator that picks which rows are used in training.
-<11> The optimization objective to target when assigning class labels. Defaults to maximize_minimum_recall.
-<12> The number of top classes to be reported in the results. Defaults to 2.
+<11> The number of top classes to be reported in the results. Defaults to 2.
 
 ===== Regression
 
diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc
index ff6ed7e2b2416..5bb4c3260ac9d 100644
--- a/docs/plugins/integrations.asciidoc
+++ b/docs/plugins/integrations.asciidoc
@@ -145,8 +145,8 @@ releases 2.0 and later do not support rivers.
 * https://micronaut-projects.github.io/micronaut-elasticsearch/latest/guide/index.html[Micronaut Elasticsearch Integration]:
   Integration of Micronaut with Elasticsearch
 
-* https://streampipes.apache.org[Apache StreamPipes]:
-  StreamPipes is a framework that enables users to work with IoT data streams allowing to store data in Elasticsearch.
+* https://docs.streampipes.org/docs/user-guide-introduction[StreamPipes]:
+  StreamPipes is a framework that enables users to work with data streams allowing to store data in Elasticsearch.
 
 * https://metamodel.apache.org/[Apache MetaModel]:
   Providing a common interface for discovery, exploration of metadata and querying of different types of data sources.
diff --git a/docs/python/index.asciidoc b/docs/python/index.asciidoc
index 61a1570ce585b..8def003e7c2ab 100644
--- a/docs/python/index.asciidoc
+++ b/docs/python/index.asciidoc
@@ -5,12 +5,12 @@
 Official low-level client for Elasticsearch. Its goal is to provide common
 ground for all Elasticsearch-related code in Python; because of this it tries
 to be opinion-free and very extendable. The full documentation is available at
-http://elasticsearch-py.readthedocs.org/
+http://elasticsearch-py.rtfd.org/
 
 .Elasticsearch DSL
 ************************************************************************************
 For a more high level client library with more limited scope, have a look at
-http://elasticsearch-dsl.readthedocs.org/[elasticsearch-dsl] - a more pythonic library
+http://elasticsearch-dsl.rtfd.org/[elasticsearch-dsl] - a more pythonic library
 sitting on top of `elasticsearch-py`.
 
 It provides a more convenient and idiomatic way to write and manipulate
diff --git a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
index d6c31d54c9550..8d4652aebe2c5 100644
--- a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
@@ -286,7 +286,7 @@ precision::     Optional. The string length of the geohashes used to define
                 to precision levels higher than the supported 12 levels,
                 (e.g. for distances <5.6cm) the value is rejected.
 
-bounds::        Optional. The bounding box to filter the points in the bucket.
+bounds:         Optional. The bounding box to filter the points in the bucket.
 
 size::          Optional. The maximum number of geohash buckets to return
                 (defaults to 10,000). When results are trimmed, buckets are
diff --git a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
index 5958d09f4ed71..cee0b5f7c0f22 100644
--- a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
@@ -70,27 +70,23 @@ the same sort values then this aggregation could return either document's fields
 
 ==== `metrics`
 
-`metrics` selects the fields to of the "top" document to return.
+`metrics` selects the fields to of the "top" document to return. Like most other
+aggregations, `top_metrics` casts these values cast to `double` precision
+floating point numbers. So they have to be numeric. Dates *work*, but they
+come back as a `double` precision floating point containing milliseconds since
+epoch. `keyword` fields aren't allowed.
 
 You can return multiple metrics by providing a list:
 
 [source,console,id=search-aggregations-metrics-top-metrics-list-of-metrics]
 ----
-PUT /test
-{
-  "mappings": {
-    "properties": {
-      "d": {"type": "date"}
-    }
-  }
-}
 POST /test/_bulk?refresh
 {"index": {}}
-{"s": 1, "v": 3.1415, "m": 1, "d": "2020-01-01T00:12:12Z"}
+{"s": 1, "v": 3.1415, "m": 1.9}
 {"index": {}}
-{"s": 2, "v": 1.0, "m": 6, "d": "2020-01-02T00:12:12Z"}
+{"s": 2, "v": 1.0, "m": 6.7}
 {"index": {}}
-{"s": 3, "v": 2.71828, "m": -12, "d": "2019-12-31T00:12:12Z"}
+{"s": 3, "v": 2.71828, "m": -12.2}
 POST /test/_search?filter_path=aggregations
 {
   "aggs": {
@@ -98,8 +94,7 @@ POST /test/_search?filter_path=aggregations
       "top_metrics": {
         "metrics": [
           {"field": "v"},
-          {"field": "m"},
-          {"field": "d"}
+          {"field": "m"}
         ],
         "sort": {"s": "desc"}
       }
@@ -119,8 +114,7 @@ Which returns:
         "sort": [3],
         "metrics": {
           "v": 2.718280076980591,
-          "m": -12,
-          "d": "2019-12-31T00:12:12.000Z"
+          "m": -12.199999809265137
         }
       } ]
     }
@@ -129,6 +123,7 @@ Which returns:
 ----
 // TESTRESPONSE
 
+
 ==== `size`
 
 `top_metrics` can return the top few document's worth of metrics using the size parameter:
@@ -251,14 +246,14 @@ Which returns:
           "key": "192.168.0.1",
           "doc_count": 2,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2.0 } } ]
           }
         },
         {
           "key": "192.168.0.2",
           "doc_count": 1,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3.0 } } ]
           }
         }
       ],
@@ -308,14 +303,14 @@ Which returns:
           "key": "192.168.0.2",
           "doc_count": 1,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3.0 } } ]
           }
         },
         {
           "key": "192.168.0.1",
           "doc_count": 2,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2.0 } } ]
           }
         }
       ],
diff --git a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
index d75ce6f003979..bcff83c5e9950 100644
--- a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
@@ -4,6 +4,8 @@
 <titleabbrev>Flatten graph</titleabbrev>
 ++++
 
+experimental[This functionality is marked as experimental in Lucene]
+
 The `flatten_graph` token filter accepts an arbitrary graph token
 stream, such as that produced by
 <<analysis-synonym-graph-tokenfilter>>, and flattens it into a single
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
index 8581d8cb7ec17..65cdd2575be20 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
@@ -4,496 +4,105 @@
 <titleabbrev>Word delimiter graph</titleabbrev>
 ++++
 
-Splits tokens at non-alphanumeric characters. The `word_delimiter_graph` filter
-also performs optional token normalization based on a set of rules. By default,
-the filter uses the following rules:
+experimental[This functionality is marked as experimental in Lucene]
 
-* Split tokens at non-alphanumeric characters.
-  The filter uses these characters as delimiters.
-  For example: `Super-Duper` -> `Super`, `Duper`
-* Remove leading or trailing delimiters from each token.
-  For example: `XL---42+'Autocoder'` -> `XL`, `42`, `Autocoder`
-* Split tokens at letter case transitions.
-  For example: `PowerShot` -> `Power`, `Shot`
-* Split tokens at letter-number transitions.
-  For example: `XL500` -> `XL`, `500`
-* Remove the English possessive (`'s`) from the end of each token.
-  For example: `Neil's` -> `Neil`
+Named `word_delimiter_graph`, it splits words into subwords and performs
+optional transformations on subword groups. Words are split into
+subwords with the following rules:
 
-The `word_delimiter_graph` filter uses Lucene's
-{lucene-analysis-docs}/miscellaneous/WordDelimiterGraphFilter.html[WordDelimiterGraphFilter].
+* split on intra-word delimiters (by default, all non alpha-numeric
+characters).
+* "Wi-Fi" -> "Wi", "Fi"
+* split on case transitions: "PowerShot" -> "Power", "Shot"
+* split on letter-number transitions: "SD500" -> "SD", "500"
+* leading and trailing intra-word delimiters on each subword are
+ignored: "//hello---there, 'dude'" -> "hello", "there", "dude"
+* trailing "'s" are removed for each subword: "O'Neil's" -> "O", "Neil"
 
-[TIP]
-====
-The `word_delimiter_graph` filter was designed to remove punctuation from
-complex identifiers, such as product IDs or part numbers. For these use cases,
-we recommend using the `word_delimiter_graph` filter with the
-<<analysis-keyword-tokenizer,`keyword`>> tokenizer.
+Unlike the `word_delimiter`, this token filter correctly handles positions for
+multi terms expansion at search-time when any of the following options
+are set to true:
 
-Avoid using the `word_delimiter_graph` filter to split hyphenated words, such as
-`wi-fi`. Because users often search for these words both with and without
-hyphens, we recommend using the
-<<analysis-synonym-graph-tokenfilter,`synonym_graph`>> filter instead.
-====
+ * `preserve_original`
+ * `catenate_numbers`
+ * `catenate_words`
+ * `catenate_all`
 
-[[analysis-word-delimiter-graph-tokenfilter-analyze-ex]]
-==== Example
+Parameters include:
 
-The following <<indices-analyze,analyze API>> request uses the
-`word_delimiter_graph` filter to split `Neil's-Super-Duper-XL500--42+AutoCoder`
-into normalized tokens using the filter's default rules:
-
-[source,console]
-----
-GET /_analyze
-{
-  "tokenizer": "keyword",
-  "filter": [ "word_delimiter_graph" ],
-  "text": "Neil's-Super-Duper-XL500--42+AutoCoder"
-}
-----
-
-The filter produces the following tokens:
-
-[source,txt]
-----
-[ Neil, Super, Duper, XL, 500, 42, Auto, Coder ]
-----
-
-////
-[source,console-result]
-----
-{
-  "tokens": [
-    {
-      "token": "Neil",
-      "start_offset": 0,
-      "end_offset": 4,
-      "type": "word",
-      "position": 0
-    },
-    {
-      "token": "Super",
-      "start_offset": 7,
-      "end_offset": 12,
-      "type": "word",
-      "position": 1
-    },
-    {
-      "token": "Duper",
-      "start_offset": 13,
-      "end_offset": 18,
-      "type": "word",
-      "position": 2
-    },
-    {
-      "token": "XL",
-      "start_offset": 19,
-      "end_offset": 21,
-      "type": "word",
-      "position": 3
-    },
-    {
-      "token": "500",
-      "start_offset": 21,
-      "end_offset": 24,
-      "type": "word",
-      "position": 4
-    },
-    {
-      "token": "42",
-      "start_offset": 26,
-      "end_offset": 28,
-      "type": "word",
-      "position": 5
-    },
-    {
-      "token": "Auto",
-      "start_offset": 29,
-      "end_offset": 33,
-      "type": "word",
-      "position": 6
-    },
-    {
-      "token": "Coder",
-      "start_offset": 33,
-      "end_offset": 38,
-      "type": "word",
-      "position": 7
-    }
-  ]
-}
-----
-////
-
-[[analysis-word-delimiter-graph-tokenfilter-analyzer-ex]]
-==== Add to an analyzer
-
-The following <<indices-create-index,create index API>> request uses the
-`word_delimiter_graph` filter to configure a new
-<<analysis-custom-analyzer,custom analyzer>>.
-
-[source,console]
-----
-PUT /my_index
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "keyword",
-          "filter": [ "word_delimiter_graph" ]
-        }
-      }
-    }
-  }
-}
-----
-
-[WARNING]
-====
-Avoid using the `word_delimiter_graph` filter with tokenizers that remove
-punctuation, such as the <<analysis-standard-tokenizer,`standard`>> tokenizer.
-This could prevent the `word_delimiter_graph` filter from splitting tokens
-correctly. It can also interfere with the filter's configurable parameters, such
-as <<word-delimiter-graph-tokenfilter-catenate-all,`catenate_all`>> or
-<<word-delimiter-graph-tokenfilter-preserve-original,`preserve_original`>>. We
-recommend using the <<analysis-keyword-tokenizer,`keyword`>> or
-<<analysis-whitespace-tokenizer,`whitespace`>> tokenizer instead.
-====
-
-[[word-delimiter-graph-tokenfilter-configure-parms]]
-==== Configurable parameters
-
-[[word-delimiter-graph-tokenfilter-adjust-offsets]]
-`adjust_offsets`::
-+
---
-(Optional, boolean)
-If `true`, the filter adjusts the offsets of split or catenated tokens to better
-reflect their actual position in the token stream. Defaults to `true`.
-
-[WARNING]
-====
-Set `adjust_offsets` to `false` if your analyzer uses filters, such as the
-<<analysis-trim-tokenfilter,`trim`>> filter, that change the length of tokens
-without changing their offsets. Otherwise, the `word_delimiter_graph` filter
-could produce tokens with illegal offsets.
-====
---
-
-[[word-delimiter-graph-tokenfilter-catenate-all]]
-`catenate_all`::
-+
---
-(Optional, boolean)
-If `true`, the filter produces catenated tokens for chains of alphanumeric
-characters separated by non-alphabetic delimiters. For example:
-`super-duper-xl-500` -> [ **`superduperxl500`**, `super`, `duper`, `xl`, `500` ].
-Defaults to `false`.
-
-[WARNING]
-====
-Setting this parameter to `true` produces multi-position tokens, which are not
-supported by indexing.
-
-If this parameter is `true`, avoid using this filter in an index analyzer or
-use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
-this filter to make the token stream suitable for indexing.
-
-When used for search analysis, catenated tokens can cause problems for the
-<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
-rely on token position for matching. Avoid setting this parameter to `true` if
-you plan to use these queries.
-====
---
-
-[[word-delimiter-graph-tokenfilter-catenate-numbers]]
-`catenate_numbers`::
-+
---
-(Optional, boolean)
-If `true`, the filter produces catenated tokens for chains of numeric characters
-separated by non-alphabetic delimiters. For example: `01-02-03` ->
-[ **`010203`**, `01`, `02`, `03` ]. Defaults to `false`.
-
-[WARNING]
-====
-Setting this parameter to `true` produces multi-position tokens, which are not
-supported by indexing.
-
-If this parameter is `true`, avoid using this filter in an index analyzer or
-use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
-this filter to make the token stream suitable for indexing.
+`generate_word_parts`::
+    If `true` causes parts of words to be
+    generated: "PowerShot" -> "Power" "Shot". Defaults to `true`.
 
-When used for search analysis, catenated tokens can cause problems for the
-<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
-rely on token position for matching. Avoid setting this parameter to `true` if
-you plan to use these queries.
-====
---
+`generate_number_parts`::
+    If `true` causes number subwords to be
+    generated: "500-42" -> "500" "42". Defaults to `true`.
 
-[[word-delimiter-graph-tokenfilter-catenate-words]]
 `catenate_words`::
-+
---
-(Optional, boolean)
-If `true`, the filter produces catenated tokens for chains of alphabetical
-characters separated by non-alphabetic delimiters. For example: `super-duper-xl`
--> [ **`superduperxl`**, `super`, `duper`, `xl` ]. Defaults to `false`.
-
-[WARNING]
-====
-Setting this parameter to `true` produces multi-position tokens, which are not
-supported by indexing.
-
-If this parameter is `true`, avoid using this filter in an index analyzer or
-use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
-this filter to make the token stream suitable for indexing.
+    If `true` causes maximum runs of word parts to be
+    catenated: "wi-fi" -> "wifi". Defaults to `false`.
 
-When used for search analysis, catenated tokens can cause problems for the
-<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
-rely on token position for matching. Avoid setting this parameter to `true` if
-you plan to use these queries.
-====
---
+`catenate_numbers`::
+    If `true` causes maximum runs of number parts to
+    be catenated: "500-42" -> "50042". Defaults to `false`.
 
-`generate_number_parts`::
-(Optional, boolean)
-If `true`, the filter includes tokens consisting of only numeric characters in
-the output. If `false`, the filter excludes these tokens from the output.
-Defaults to `true`.
+`catenate_all`::
+    If `true` causes all subword parts to be catenated:
+    "wi-fi-4000" -> "wifi4000". Defaults to `false`.
 
-`generate_word_parts`::
-(Optional, boolean)
-If `true`, the filter includes tokens consisting of only alphabetical characters
-in the output. If `false`, the filter excludes these tokens from the output.
-Defaults to `true`.
+`split_on_case_change`::
+    If `true` causes "PowerShot" to be two tokens;
+    ("Power-Shot" remains two parts regards). Defaults to `true`.
 
-[[word-delimiter-graph-tokenfilter-preserve-original]]
 `preserve_original`::
-+
---
-(Optional, boolean)
-If `true`, the filter includes the original version of any split tokens in the
-output. This original version includes non-alphanumeric delimiters. For example:
-`super-duper-xl-500` -> [ **`super-duper-xl-500`**, `super`, `duper`, `xl`,
-`500` ]. Defaults to `false`.
-
-[WARNING]
-====
-Setting this parameter to `true` produces multi-position tokens, which are not
-supported by indexing.
-
-If this parameter is `true`, avoid using this filter in an index analyzer or
-use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
-this filter to make the token stream suitable for indexing.
-====
---
-
-`protected_words`::
-(Optional, array of strings)
-Array of tokens the filter won't split.
-
-`protected_words_path`::
-+
---
-(Optional, string)
-Path to a file that contains a list of tokens the filter won't split.
-
-This path must be absolute or relative to the `config` location, and the file
-must be UTF-8 encoded. Each token in the file must be separated by a line
-break.
---
-
-`split_on_case_change`::
-(Optional, boolean)
-If `true`, the filter splits tokens at letter case transitions. For example:
-`camelCase` -> [ `camel`, `Case` ]. Defaults to `true`.
+    If `true` includes original words in subwords:
+    "500-42" -> "500-42" "500" "42". Defaults to `false`.
 
 `split_on_numerics`::
-(Optional, boolean)
-If `true`, the filter splits tokens at letter-number transitions. For example:
-`j2se` -> [ `j`, `2`, `se` ]. Defaults to `true`.
+    If `true` causes "j2se" to be three tokens; "j"
+    "2" "se". Defaults to `true`.
 
 `stem_english_possessive`::
-(Optional, boolean)
-If `true`, the filter removes the English possessive (`'s`) from the end of each
-token. For example: `O'Neil's` -> [ `O`, `Neil` ]. Defaults to `true`.
+    If `true` causes trailing "'s" to be
+    removed for each subword: "O'Neil's" -> "O", "Neil". Defaults to `true`.
 
-`type_table`::
-+
---
-(Optional, array of strings)
-Array of custom type mappings for characters. This allows you to map
-non-alphanumeric characters as numeric or alphanumeric to avoid splitting on
-those characters.
+Advance settings include:
 
-For example, the following array maps the plus (`+`) and hyphen (`-`) characters
-as alphanumeric, which means they won't be treated as delimiters:
-
-`[ "+ => ALPHA", "- => ALPHA" ]`
-
-Supported types include:
-
-* `ALPHA` (Alphabetical)
-* `ALPHANUM` (Alphanumeric)
-* `DIGIT` (Numeric)
-* `LOWER` (Lowercase alphabetical)
-* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
-* `UPPER` (Uppercase alphabetical)
---
-
-`type_table_path`::
-+
---
-(Optional, string)
-Path to a file that contains custom type mappings for characters. This allows
-you to map non-alphanumeric characters as numeric or alphanumeric to avoid
-splitting on those characters.
-
-For example, the contents of this file may contain the following:
-
-[source,txt]
-----
-# Map the $, %, '.', and ',' characters to DIGIT
-# This might be useful for financial data.
-$ => DIGIT
-% => DIGIT
-. => DIGIT
-\\u002C => DIGIT
-
-# in some cases you might not want to split on ZWJ
-# this also tests the case where we need a bigger byte[]
-# see http://en.wikipedia.org/wiki/Zero-width_joiner
-\\u200D => ALPHANUM
-----
-
-Supported types include:
-
-* `ALPHA` (Alphabetical)
-* `ALPHANUM` (Alphanumeric)
-* `DIGIT` (Numeric)
-* `LOWER` (Lowercase alphabetical)
-* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
-* `UPPER` (Uppercase alphabetical)
-
-This file path must be absolute or relative to the `config` location, and the
-file must be UTF-8 encoded. Each mapping in the file must be separated by a line
-break.
---
-
-[[analysis-word-delimiter-graph-tokenfilter-customize]]
-==== Customize
-
-To customize the `word_delimiter_graph` filter, duplicate it to create the basis
-for a new custom token filter. You can modify the filter using its configurable
-parameters.
-
-For example, the following request creates a `word_delimiter_graph`
-filter that uses the following rules:
-
-* Split tokens at non-alphanumeric characters, _except_ the hyphen (`-`)
-  character.
-* Remove leading or trailing delimiters from each token.
-* Do _not_ split tokens at letter case transitions.
-* Do _not_ split tokens at letter-number transitions.
-* Remove the English possessive (`'s`) from the end of each token.
-
-[source,console]
-----
-PUT /my_index
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "keyword",
-          "filter": [ "my_custom_word_delimiter_graph_filter" ]
-        }
-      },
-      "filter": {
-        "my_custom_word_delimiter_graph_filter": {
-          "type": "word_delimiter_graph",
-          "type_table": [ "- => ALPHA" ],
-          "split_on_case_change": false,
-          "split_on_numerics": false,
-          "stem_english_possessive": true
-        }
-      }
-    }
-  }
-}
-----
-
-[[analysis-word-delimiter-graph-differences]]
-==== Differences between `word_delimiter_graph` and `word_delimiter` 
-
-Both the  `word_delimiter_graph` and
-<<analysis-word-delimiter-tokenfilter,`word_delimiter`>> filters produce tokens
-that span multiple positions when any of the following parameters are `true`:
-
- * <<word-delimiter-graph-tokenfilter-catenate-all,`catenate_all`>>
- * <<word-delimiter-graph-tokenfilter-catenate-numbers,`catenate_numbers`>>
- * <<word-delimiter-graph-tokenfilter-catenate-words,`catenate_words`>>
- * <<word-delimiter-graph-tokenfilter-preserve-original,`preserve_original`>>
-
-However, only the `word_delimiter_graph` filter assigns multi-position tokens a
-`positionLength` attribute, which indicates the number of positions a token
-spans. This ensures the `word_delimiter_graph` filter always produces valid token
-https://en.wikipedia.org/wiki/Directed_acyclic_graph[graphs].
-
-The `word_delimiter` filter does not assign multi-position tokens a
-`positionLength` attribute. This means it produces invalid graphs for streams
-including these tokens.
-
-While indexing does not support token graphs containing multi-position tokens,
-queries, such as the <<query-dsl-match-query-phrase,`match_phrase`>> query, can
-use these graphs to generate multiple sub-queries from a single query string.
-
-To see how token graphs produced by the `word_delimiter` and
-`word_delimiter_graph` filters differ, check out the following example.
-
-.*Example*
-[%collapsible]
-====
-
-[[analysis-word-delimiter-graph-basic-token-graph]]
-*Basic token graph*
-
-Both the `word_delimiter` and `word_delimiter_graph` produce the following token
-graph for `PowerShot2000` when the following parameters are `false`:
-
- * <<word-delimiter-graph-tokenfilter-catenate-all,`catenate_all`>>
- * <<word-delimiter-graph-tokenfilter-catenate-numbers,`catenate_numbers`>>
- * <<word-delimiter-graph-tokenfilter-catenate-words,`catenate_words`>>
- * <<word-delimiter-graph-tokenfilter-preserve-original,`preserve_original`>>
-
-This graph does not contain multi-position tokens. All tokens span only one
-position.
-
-image::images/analysis/token-graph-basic.svg[align="center"]
-
-[[analysis-word-delimiter-graph-wdg-token-graph]]
-*`word_delimiter_graph` graph with a multi-position token*
-
-The `word_delimiter_graph` filter produces the following token graph for
-`PowerShot2000` when `catenate_words` is `true`.
-
-This graph correctly indicates the catenated `PowerShot` token spans two
-positions.
-
-image::images/analysis/token-graph-wdg.svg[align="center"]
-
-[[analysis-word-delimiter-graph-wd-token-graph]]
-*`word_delimiter` graph with a multi-position token*
-
-When `catenate_words` is `true`, the `word_delimiter` filter produces
-the following token graph for `PowerShot2000`.
-
-Note that the catenated `PowerShot` token should span two positions but only
-spans one in the token graph, making it invalid.
+`protected_words`::
+    A list of protected words from being delimiter.
+    Either an array, or also can set `protected_words_path` which resolved
+    to a file configured with protected words (one on each line).
+    Automatically resolves to `config/` based location if exists.
 
-image::images/analysis/token-graph-wd.svg[align="center"]
+`adjust_offsets`::
+    By default, the filter tries to output subtokens with adjusted offsets
+    to reflect their actual position in the token stream.  However, when
+    used in combination with other filters that alter the length or starting
+    position of tokens without changing their offsets
+    (e.g. <<analysis-trim-tokenfilter,`trim`>>) this can cause tokens with
+    illegal offsets to be emitted.  Setting `adjust_offsets` to false will
+    stop `word_delimiter_graph` from adjusting these internal offsets.
 
-====
\ No newline at end of file
+`type_table`::
+    A custom type mapping table, for example (when configured
+    using `type_table_path`):
+
+[source,type_table]
+--------------------------------------------------
+    # Map the $, %, '.', and ',' characters to DIGIT
+    # This might be useful for financial data.
+    $ => DIGIT
+    % => DIGIT
+    . => DIGIT
+    \\u002C => DIGIT
+
+    # in some cases you might not want to split on ZWJ
+    # this also tests the case where we need a bigger byte[]
+    # see http://en.wikipedia.org/wiki/Zero-width_joiner
+    \\u200D => ALPHANUM
+--------------------------------------------------
+
+NOTE: Using a tokenizer like the `standard` tokenizer may interfere with
+the `catenate_*` and `preserve_original` parameters, as the original
+string may already have lost punctuation during tokenization.  Instead,
+you may want to use the `whitespace` tokenizer.
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
index 02d6257cb4ee9..25074b2725ea4 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
@@ -4,379 +4,84 @@
 <titleabbrev>Word delimiter</titleabbrev>
 ++++
 
-[WARNING]
-====
-We recommend using the
-<<analysis-word-delimiter-graph-tokenfilter,`word_delimiter_graph`>> instead of
-the `word_delimiter` filter.
+Named `word_delimiter`, it Splits words into subwords and performs
+optional transformations on subword groups. Words are split into
+subwords with the following rules:
 
-The `word_delimiter` filter can produce invalid token graphs. See
-<<analysis-word-delimiter-graph-differences>>.
+* split on intra-word delimiters (by default, all non alpha-numeric
+characters): "Wi-Fi" -> "Wi", "Fi"
+* split on case transitions: "PowerShot" -> "Power", "Shot"
+* split on letter-number transitions: "SD500" -> "SD", "500"
+* leading and trailing intra-word delimiters on each subword are
+ignored: "//hello---there, 'dude'" -> "hello", "there", "dude"
+* trailing "'s" are removed for each subword: "O'Neil's" -> "O", "Neil"
 
-The `word_delimiter` filter also uses Lucene's
-{lucene-analysis-docs}/miscellaneous/WordDelimiterFilter.html[WordDelimiterFilter],
-which is marked as deprecated. 
-====
+Parameters include:
 
-Splits tokens at non-alphanumeric characters. The `word_delimiter` filter
-also performs optional token normalization based on a set of rules. By default,
-the filter uses the following rules:
-
-* Split tokens at non-alphanumeric characters.
-  The filter uses these characters as delimiters.
-  For example: `Super-Duper` -> `Super`, `Duper`
-* Remove leading or trailing delimiters from each token.
-  For example: `XL---42+'Autocoder'` -> `XL`, `42`, `Autocoder`
-* Split tokens at letter case transitions.
-  For example: `PowerShot` -> `Power`, `Shot`
-* Split tokens at letter-number transitions.
-  For example: `XL500` -> `XL`, `500`
-* Remove the English possessive (`'s`) from the end of each token.
-  For example: `Neil's` -> `Neil`
-
-[TIP]
-====
-The `word_delimiter` filter was designed to remove punctuation from complex
-identifiers, such as product IDs or part numbers. For these use cases, we
-recommend using the `word_delimiter` filter with the
-<<analysis-keyword-tokenizer,`keyword`>> tokenizer.
-
-Avoid using the `word_delimiter` filter to split hyphenated words, such as
-`wi-fi`. Because users often search for these words both with and without
-hyphens, we recommend using the
-<<analysis-synonym-graph-tokenfilter,`synonym_graph`>> filter instead.
-====
-
-[[analysis-word-delimiter-tokenfilter-analyze-ex]]
-==== Example
-
-The following <<indices-analyze,analyze API>> request uses the
-`word_delimiter` filter to split `Neil's-Super-Duper-XL500--42+AutoCoder`
-into normalized tokens using the filter's default rules:
-
-[source,console]
-----
-GET /_analyze
-{
-  "tokenizer": "keyword",
-  "filter": [ "word_delimiter" ],
-  "text": "Neil's-Super-Duper-XL500--42+AutoCoder"
-}
-----
-
-The filter produces the following tokens:
-
-[source,txt]
-----
-[ Neil, Super, Duper, XL, 500, 42, Auto, Coder ]
-----
-
-////
-[source,console-result]
-----
-{
-  "tokens": [
-    {
-      "token": "Neil",
-      "start_offset": 0,
-      "end_offset": 4,
-      "type": "word",
-      "position": 0
-    },
-    {
-      "token": "Super",
-      "start_offset": 7,
-      "end_offset": 12,
-      "type": "word",
-      "position": 1
-    },
-    {
-      "token": "Duper",
-      "start_offset": 13,
-      "end_offset": 18,
-      "type": "word",
-      "position": 2
-    },
-    {
-      "token": "XL",
-      "start_offset": 19,
-      "end_offset": 21,
-      "type": "word",
-      "position": 3
-    },
-    {
-      "token": "500",
-      "start_offset": 21,
-      "end_offset": 24,
-      "type": "word",
-      "position": 4
-    },
-    {
-      "token": "42",
-      "start_offset": 26,
-      "end_offset": 28,
-      "type": "word",
-      "position": 5
-    },
-    {
-      "token": "Auto",
-      "start_offset": 29,
-      "end_offset": 33,
-      "type": "word",
-      "position": 6
-    },
-    {
-      "token": "Coder",
-      "start_offset": 33,
-      "end_offset": 38,
-      "type": "word",
-      "position": 7
-    }
-  ]
-}
-----
-////
-
-[analysis-word-delimiter-tokenfilter-analyzer-ex]]
-==== Add to an analyzer
-
-The following <<indices-create-index,create index API>> request uses the
-`word_delimiter` filter to configure a new
-<<analysis-custom-analyzer,custom analyzer>>.
-
-[source,console]
-----
-PUT /my_index
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "keyword",
-          "filter": [ "word_delimiter" ]
-        }
-      }
-    }
-  }
-}
-----
-
-[WARNING]
-====
-Avoid using the `word_delimiter` filter with tokenizers that remove punctuation,
-such as the <<analysis-standard-tokenizer,`standard`>> tokenizer. This could
-prevent the `word_delimiter` filter from splitting tokens correctly. It can also
-interfere with the filter's configurable parameters, such as `catenate_all` or
-`preserve_original`. We recommend using the
-<<analysis-keyword-tokenizer,`keyword`>> or
-<<analysis-whitespace-tokenizer,`whitespace`>> tokenizer instead.
-====
-
-[[word-delimiter-tokenfilter-configure-parms]]
-==== Configurable parameters
-
-`catenate_all`::
-+
---
-(Optional, boolean)
-If `true`, the filter produces catenated tokens for chains of alphanumeric
-characters separated by non-alphabetic delimiters. For example:
-`super-duper-xl-500` -> [ `super`, **`superduperxl500`**, `duper`, `xl`, `500`
-]. Defaults to `false`.
-
-[WARNING]
-====
-When used for search analysis, catenated tokens can cause problems for the
-<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
-rely on token position for matching. Avoid setting this parameter to `true` if
-you plan to use these queries.
-====
---
-
-`catenate_numbers`::
-+
---
-(Optional, boolean)
-If `true`, the filter produces catenated tokens for chains of numeric characters
-separated by non-alphabetic delimiters. For example: `01-02-03` ->
-[ `01`, **`010203`**, `02`, `03` ]. Defaults to `false`.
+`generate_word_parts`::
+    If `true` causes parts of words to be
+    generated: "Power-Shot", "(Power,Shot)" -> "Power" "Shot". Defaults to `true`.
 
-[WARNING]
-====
-When used for search analysis, catenated tokens can cause problems for the
-<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
-rely on token position for matching. Avoid setting this parameter to `true` if
-you plan to use these queries.
-====
---
+`generate_number_parts`::
+    If `true` causes number subwords to be
+    generated: "500-42" -> "500" "42". Defaults to `true`.
 
 `catenate_words`::
-+
---
-(Optional, boolean)
-If `true`, the filter produces catenated tokens for chains of alphabetical
-characters separated by non-alphabetic delimiters. For example: `super-duper-xl`
--> [ `super`, **`superduperxl`**, `duper`, `xl` ]. Defaults to `false`.
+    If `true` causes maximum runs of word parts to be
+    catenated: "wi-fi" -> "wifi". Defaults to `false`.
 
-[WARNING]
-====
-When used for search analysis, catenated tokens can cause problems for the
-<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
-rely on token position for matching. Avoid setting this parameter to `true` if
-you plan to use these queries.
-====
---
+`catenate_numbers`::
+    If `true` causes maximum runs of number parts to
+    be catenated: "500-42" -> "50042". Defaults to `false`.
 
-`generate_number_parts`::
-(Optional, boolean)
-If `true`, the filter includes tokens consisting of only numeric characters in
-the output. If `false`, the filter excludes these tokens from the output.
-Defaults to `true`.
+`catenate_all`::
+    If `true` causes all subword parts to be catenated:
+    "wi-fi-4000" -> "wifi4000". Defaults to `false`.
 
-`generate_word_parts`::
-(Optional, boolean)
-If `true`, the filter includes tokens consisting of only alphabetical characters
-in the output. If `false`, the filter excludes these tokens from the output.
-Defaults to `true`.
+`split_on_case_change`::
+    If `true` causes "PowerShot" to be two tokens;
+    ("Power-Shot" remains two parts regards). Defaults to `true`.
 
 `preserve_original`::
-(Optional, boolean)
-If `true`, the filter includes the original version of any split tokens in the
-output. This original version includes non-alphanumeric delimiters. For example:
-`super-duper-xl-500` -> [ **`super-duper-xl-500`**, `super`, `duper`, `xl`,
-`500` ]. Defaults to `false`.
-
-`protected_words`::
-(Optional, array of strings)
-Array of tokens the filter won't split.
-
-`protected_words_path`::
-+
---
-(Optional, string)
-Path to a file that contains a list of tokens the filter won't split.
-
-This path must be absolute or relative to the `config` location, and the file
-must be UTF-8 encoded. Each token in the file must be separated by a line
-break.
---
-
-`split_on_case_change`::
-(Optional, boolean)
-If `true`, the filter splits tokens at letter case transitions. For example:
-`camelCase` -> [ `camel`, `Case` ]. Defaults to `true`.
+    If `true` includes original words in subwords:
+    "500-42" -> "500-42" "500" "42". Defaults to `false`.
 
 `split_on_numerics`::
-(Optional, boolean)
-If `true`, the filter splits tokens at letter-number transitions. For example:
-`j2se` -> [ `j`, `2`, `se` ]. Defaults to `true`.
+    If `true` causes "j2se" to be three tokens; "j"
+    "2" "se". Defaults to `true`.
 
 `stem_english_possessive`::
-(Optional, boolean)
-If `true`, the filter removes the English possessive (`'s`) from the end of each
-token. For example: `O'Neil's` -> [ `O`, `Neil` ]. Defaults to `true`.
-
-`type_table`::
-+
---
-(Optional, array of strings)
-Array of custom type mappings for characters. This allows you to map
-non-alphanumeric characters as numeric or alphanumeric to avoid splitting on
-those characters.
-
-For example, the following array maps the plus (`+`) and hyphen (`-`) characters
-as alphanumeric, which means they won't be treated as delimiters:
-
-`[ "+ => ALPHA", "- => ALPHA" ]`
-
-Supported types include:
+    If `true` causes trailing "'s" to be
+    removed for each subword: "O'Neil's" -> "O", "Neil". Defaults to `true`.
 
-* `ALPHA` (Alphabetical)
-* `ALPHANUM` (Alphanumeric)
-* `DIGIT` (Numeric)
-* `LOWER` (Lowercase alphabetical)
-* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
-* `UPPER` (Uppercase alphabetical)
---
+Advance settings include:
 
-`type_table_path`::
-+
---
-(Optional, string)
-Path to a file that contains custom type mappings for characters. This allows
-you to map non-alphanumeric characters as numeric or alphanumeric to avoid
-splitting on those characters.
-
-For example, the contents of this file may contain the following:
-
-[source,txt]
-----
-# Map the $, %, '.', and ',' characters to DIGIT
-# This might be useful for financial data.
-$ => DIGIT
-% => DIGIT
-. => DIGIT
-\\u002C => DIGIT
-
-# in some cases you might not want to split on ZWJ
-# this also tests the case where we need a bigger byte[]
-# see http://en.wikipedia.org/wiki/Zero-width_joiner
-\\u200D => ALPHANUM
-----
-
-Supported types include:
-
-* `ALPHA` (Alphabetical)
-* `ALPHANUM` (Alphanumeric)
-* `DIGIT` (Numeric)
-* `LOWER` (Lowercase alphabetical)
-* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
-* `UPPER` (Uppercase alphabetical)
-
-This file path must be absolute or relative to the `config` location, and the
-file must be UTF-8 encoded. Each mapping in the file must be separated by a line
-break.
---
-
-[[analysis-word-delimiter-tokenfilter-customize]]
-==== Customize
-
-To customize the `word_delimiter` filter, duplicate it to create the basis
-for a new custom token filter. You can modify the filter using its configurable
-parameters.
-
-For example, the following request creates a `word_delimiter`
-filter that uses the following rules:
-
-* Split tokens at non-alphanumeric characters, _except_ the hyphen (`-`)
-  character.
-* Remove leading or trailing delimiters from each token.
-* Do _not_ split tokens at letter case transitions.
-* Do _not_ split tokens at letter-number transitions.
-* Remove the English possessive (`'s`) from the end of each token.
+`protected_words`::
+    A list of protected words from being delimiter.
+    Either an array, or also can set `protected_words_path` which resolved
+    to a file configured with protected words (one on each line).
+    Automatically resolves to `config/` based location if exists.
 
-[source,console]
-----
-PUT /my_index
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "keyword",
-          "filter": [ "my_custom_word_delimiter_filter" ]
-        }
-      },
-      "filter": {
-        "my_custom_word_delimiter_filter": {
-          "type": "word_delimiter",
-          "type_table": [ "- => ALPHA" ],
-          "split_on_case_change": false,
-          "split_on_numerics": false,
-          "stem_english_possessive": true
-        }
-      }
-    }
-  }
-}
-----
\ No newline at end of file
+`type_table`::
+    A custom type mapping table, for example (when configured
+    using `type_table_path`):
+
+[source,type_table]
+--------------------------------------------------
+    # Map the $, %, '.', and ',' characters to DIGIT
+    # This might be useful for financial data.
+    $ => DIGIT
+    % => DIGIT
+    . => DIGIT
+    \\u002C => DIGIT
+
+    # in some cases you might not want to split on ZWJ
+    # this also tests the case where we need a bigger byte[]
+    # see http://en.wikipedia.org/wiki/Zero-width_joiner
+    \\u200D => ALPHANUM
+--------------------------------------------------
+
+NOTE: Using a tokenizer like the `standard` tokenizer may interfere with
+the `catenate_*` and `preserve_original` parameters, as the original
+string may already have lost punctuation during tokenization.  Instead,
+you may want to use the `whitespace` tokenizer.
diff --git a/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
index 67c0cefc98957..5da001640a027 100644
--- a/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
@@ -1,6 +1,8 @@
 [[analysis-simplepattern-tokenizer]]
 === Simple Pattern Tokenizer
 
+experimental[This functionality is marked as experimental in Lucene]
+
 The `simple_pattern` tokenizer uses a regular expression to capture matching
 text as terms. The set of regular expression features it supports is more
 limited than the <<analysis-pattern-tokenizer,`pattern`>> tokenizer, but the
diff --git a/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
index 3f24233334e57..55be14c45638a 100644
--- a/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
@@ -1,6 +1,8 @@
 [[analysis-simplepatternsplit-tokenizer]]
 === Simple Pattern Split Tokenizer
 
+experimental[This functionality is marked as experimental in Lucene]
+
 The `simple_pattern_split` tokenizer uses a regular expression to split the
 input into terms at pattern matches. The set of regular expression features it
 supports is more limited than the <<analysis-pattern-tokenizer,`pattern`>>
diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc
index 05931ea2d6a16..d7eff24e576d8 100644
--- a/docs/reference/cluster/remote-info.asciidoc
+++ b/docs/reference/cluster/remote-info.asciidoc
@@ -17,20 +17,25 @@ Returns configured remote cluster information.
 ==== {api-description-title}
 
 The cluster remote info API allows you to retrieve all of the configured
-remote cluster information. It returns connection and endpoint information keyed
+remote cluster information. It returns connection and endpoint information keyed 
 by the configured remote cluster alias.
 
 
 [[cluster-remote-info-api-response-body]]
 ==== {api-response-body-title}
 
-`mode`::
-    Connection mode for the remote cluster. Returned values are `sniff` and
-    `proxy`.
+`seeds`::
+	The configured initial seed transport addresses of the remote cluster.
 
 `connected`::
 	True if there is at least one connection to the remote cluster.
 
+`num_nodes_connected`::
+    The number of connected nodes in the remote cluster.
+
+`max_connections_per_cluster`::
+	The maximum number of connections maintained for the remote cluster.
+
 `initial_connect_timeout`::
 	The initial connect timeout for remote cluster connections.
 
@@ -38,26 +43,3 @@ by the configured remote cluster alias.
 `skip_unavailable`::
     Whether the remote cluster is skipped in case it is searched through
     a {ccs} request but none of its nodes are available.
-
-`seeds`::
-    Initial seed transport addresses of the remote cluster when sniff mode is
-    configured.
-
-`num_nodes_connected`::
-    Number of connected nodes in the remote cluster when sniff mode is
-    configured.
-
-`max_connections_per_cluster`::
-    Maximum number of connections maintained for the remote cluster when sniff
-    mode is configured.
-
-`proxy_address`::
-    Address for remote connections when proxy mode is configured.
-
-`num_proxy_sockets_connected`::
-    Number of open socket connections to the remote cluster when proxy mode
-    is configured.
-
-`max_proxy_socket_connections`::
-    The maximum number of socket connections to the remote cluster when proxy
-    mode is configured.
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index 9ae24b17eaae1..cc9b41f8b96df 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -18,9 +18,9 @@ Returns cluster statistics.
 [[cluster-stats-api-desc]]
 ==== {api-description-title}
 
-The Cluster Stats API allows to retrieve statistics from a cluster wide
-perspective. The API returns basic index metrics (shard numbers, store size,
-memory usage) and information about the current nodes that form the cluster
+The Cluster Stats API allows to retrieve statistics from a cluster wide 
+perspective. The API returns basic index metrics (shard numbers, store size, 
+memory usage) and information about the current nodes that form the cluster 
 (number, roles, os, jvm versions, memory usage, cpu and installed plugins).
 
 
@@ -72,16 +72,6 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=cluster-health-status]
 +
 See <<cluster-health>>.
 
-[NOTE]
-====
-The remaining statistics are grouped by section.
-====
-
-[[cluster-stats-api-response-body-indices]]
-===== `indices` section
-
-[%collapsible]
-====
 `indices.count`::
 (integer)
 Total number of indices with shards assigned to selected nodes.
@@ -348,290 +338,7 @@ This object is not populated by the cluster stats API.
 +
 To get information on segment files, use the <<cluster-nodes-stats,node stats
 API>>.
-====
-
-[[cluster-stats-api-response-body-nodes]]
-===== `nodes` section
-
-[%collapsible]
-====
-`nodes.count.total`::
-(integer)
-Total number of nodes selected by the request's <<cluster-nodes,node filters>>.
-
-`nodes.count.coordinating_only`::
-(integer)
-Number of selected nodes without a <<modules-node,role>>. These nodes are
-considered <<coordinating-only-node,coordinating only>> nodes.
-
-`nodes.count.<role>`::
-(integer)
-Number of selected nodes with the role. For a list of roles, see
-<<modules-node>>.
-
-`nodes.versions`::
-(array of strings)
-Array of {es} versions used on selected nodes.
-
-`nodes.os.available_processors`::
-(integer)
-Number of processors available to JVM across all selected nodes.
-
-`nodes.os.allocated_processors`::
-(integer)
-Number of processors used to calculate thread pool size across all selected
-nodes.
-+
-This number can be set with the `processors` setting of a node and defaults to
-the number of processors reported by the OS. In both cases, this number will
-never be larger than `32`.
-
-`nodes.os.names`::
-(array of objects)
-Contains details about operating systems used by one or more selected nodes.
-
-`nodes.os.names.name`:::
-(string)
-Name of an operating system used by one or more selected nodes.
-
-`nodes.os.names.count`:::
-(string)
-Number of selected nodes using the operating system.
-
-`nodes.os.pretty_names`::
-(array of objects)
-Contains details about operating systems used by one or more selected nodes.
-
-`nodes.os.pretty_names.pretty_name`:::
-(string)
-Human-readable name of an operating system used by one or more selected nodes.
-
-`nodes.os.pretty_names.count`:::
-(string)
-Number of selected nodes using the operating system.
-
-`nodes.os.mem.total`::
-(<<byte-units, byte units>>)
-Total amount of physical memory across all selected nodes.
-
-`nodes.os.mem.total_in_bytes`::
-(integer)
-Total amount, in bytes, of physical memory across all selected nodes.
-
-`nodes.os.mem.free`::
-(<<byte-units, byte units>>)
-Amount of free physical memory across all selected nodes.
-
-`nodes.os.mem.free_in_bytes`::
-(integer)
-Amount, in bytes, of free physical memory across all selected nodes.
-
-`nodes.os.mem.used`::
-(<<byte-units, byte units>>)
-Amount of physical memory in use across all selected nodes.
-
-`nodes.os.mem.used_in_bytes`::
-(integer)
-Amount, in bytes, of physical memory in use across all selected nodes.
-
-`nodes.os.mem.free_percent`::
-(integer)
-Percentage of free physical memory across all selected nodes.
-
-`nodes.os.mem.used_percent`::
-(integer)
-Percentage of physical memory in use across all selected nodes.
-
-`nodes.process.cpu.percent`::
-(integer)
-Percentage of CPU used across all selected nodes. Returns `-1` if
-not supported.
-
-`nodes.process.open_file_descriptors.min`::
-(integer)
-Minimum number of concurrently open file descriptors across all selected nodes.
-Returns `-1` if not supported.
-
-`nodes.process.open_file_descriptors.max`::
-(integer)
-Maximum number of concurrently open file descriptors allowed across all selected
-nodes. Returns `-1` if not supported.
-
-`nodes.process.open_file_descriptors.avg`::
-(integer)
-Average number of concurrently open file descriptors. Returns `-1` if not
-supported.
-
-`nodes.jvm.max_uptime`::
-(<<time-units,time unit>>)
-Uptime duration since JVM last started.
-
-`nodes.jvm.max_uptime_in_millis`::
-(integer)
-Uptime duration, in milliseconds, since JVM last started.
-
-`nodes.jvm.versions`::
-(array of objects)
-Contains details about the JVM versions used by selected
-nodes.
-
-`nodes.jvm.versions.version`:::
-(string)
-Version of JVM used by one or more selected nodes.
-
-`nodes.jvm.versions.vm_name`:::
-(string)
-Name of the JVM.
-
-`nodes.jvm.versions.vm_version`:::
-(string)
-Full version number of JVM.
-+
-The full version number includes a plus sign (`+`) followed by the build number.
 
-`nodes.jvm.versions.vm_vendor`:::
-(string)
-Vendor of the JVM.
-
-`nodes.jvm.versions.bundled_jdk`:::
-(boolean)
-If `true`, the JVM includes a bundled Java Development Kit (JDK).
-
-`nodes.jvm.versions.using_bundled_jdk`:::
-(boolean)
-If `true`, a bundled JDK is in use by JVM.
-
-`nodes.jvm.versions.count`:::
-(integer)
-Total number of selected nodes using JVM.
-
-`nodes.jvm.mem.heap_used`::
-(<<byte-units, byte units>>)
-Memory currently in use by the heap across all selected nodes.
-
-`nodes.jvm.mem.heap_used_in_bytes`::
-(integer)
-Memory, in bytes, currently in use by the heap across all selected nodes.
-
-`nodes.jvm.mem.heap_max`::
-(<<byte-units, byte units>>)
-Maximum amount of memory, in bytes, available for use by the heap across all
-selected nodes.
-
-`nodes.jvm.mem.heap_max_in_bytes`::
-(integer)
-Maximum amount of memory, in bytes, available for use by the heap across all
-selected nodes.
-
-`nodes.jvm.threads`::
-(integer)
-Number of active threads in use by JVM across all selected nodes.
-
-`nodes.fs.total`::
-(<<byte-units, byte units>>)
-Total size of all file stores across all selected nodes.
-
-`nodes.fs.total_in_bytes`::
-(integer)
-Total size, in bytes, of all file stores across all seleced nodes.
-
-`nodes.fs.free`::
-(<<byte-units, byte units>>)
-Amount of unallocated disk space in file stores across all selected nodes.
-
-`nodes.fs.free_in_bytes`::
-(integer)
-Total number of unallocated bytes in file stores across all selected nodes.
-
-`nodes.fs.available`::
-(<<byte-units, byte units>>)
-Total amount of disk space available to JVM in file
-stores across all selected nodes.
-+
-Depending on OS or process-level restrictions, this amount may be less than
-`nodes.fs.free`. This is the actual amount of free disk space the selected {es}
-nodes can use.
-
-`nodes.fs.available_in_bytes`::
-(integer)
-Total number of bytes available to JVM in file stores
-across all selected nodes.
-+
-Depending on OS or process-level restrictions, this number may be less than
-`nodes.fs.free_in_byes`. This is the actual amount of free disk space the
-selected {es} nodes can use.
-
-`nodes.plugins`::
-(array of objects)
-Contains details about installed plugins and modules across all selected nodes.
-+
-If no plugins or modules are installed, this array is empty.
-
-`nodes.plugins.name`:::
-(string)
-Name of the {es} plugin.
-
-`nodes.plugins.version`:::
-(string)
-{es} version for which the plugin was built.
-
-`nodes.plugins.elasticsearch_version`:::
-(string)
-{es} version for which the plugin was built.
-
-`node.plugins.java_version`:::
-(string)
-Java version for which the plugin was built.
-
-`nodes.plugins.description`:::
-(string)
-Short description of the plugin.
-
-`nodes.plugins.classname`:::
-(string)
-Class name used as the plugin's entry point.
-
-`nodes.plugins.extended_plugins`:::
-(array of strings)
-An array of other plugins extended by this plugin through the Java Service
-Provider Interface (SPI).
-+
-If this plugin extends no other plugins, this array is empty.
-
-`nodes.plugins.has_native_controller`:::
-(boolean)
-If `true`, the plugin has a native controller process.
-
-`nodes.network_types.transport_types.<transport_type>`::
-(integer)
-Number of selected nodes using the transport type.
-
-`nodes.network_types.http_types.<http_type>`::
-(integer)
-Number of selected nodes using the HTTP type.
-
-`nodes.discovery_types.<discovery_type>`::
-(integer)
-Number of selected nodes using the <<modules-discovery-hosts-providers,discovery
-type>> to find other nodes.
-
-`nodes.packaging_types`::
-(array of objects)
-Contains details about {es} distributions installed on selected nodes.
-
-`nodes.packaging_types.flavor`:::
-(string)
-Type of {es} distribution, such as `default` or `OSS`, used by one or more
-selected nodes.
-
-`nodes.packaging_types.type`:::
-(string)
-File type, such as `tar` or `zip`, used for the distribution package.
-
-`nodes.packaging_types.count`:::
-(integer)
-Number of selected nodes using the distribution flavor and file type.
-====
 
 [[cluster-stats-api-example]]
 ==== {api-examples-title}
diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc
index ccb3752854e8b..190734198c77d 100644
--- a/docs/reference/eql/search.asciidoc
+++ b/docs/reference/eql/search.asciidoc
@@ -29,15 +29,6 @@ PUT sec_logs/_bulk?refresh
 ----
 // TESTSETUP
 
-[TIP]
-====
-You also can set up {beats-ref}/getting-started.html[{beats}], such as
-{auditbeat-ref}/auditbeat-getting-started.html[{auditbeat}] or
-{winlogbeat-ref}/winlogbeat-getting-started.html[{winlogbeat}], to automatically
-send and index your event data in {es}. See
-{beats-ref}/getting-started.html[Getting started with {beats}].
-====
-
 You can now use the EQL search API to search this index using an EQL query.
 
 The following request searches the `sec_logs` index using the EQL query
diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc
index fac44f8bcdc53..40ff0828afe2d 100644
--- a/docs/reference/glossary.asciidoc
+++ b/docs/reference/glossary.asciidoc
@@ -24,55 +24,28 @@ Also see <<glossary-text,text>> and <<glossary-term,term>>.
 // end::analysis-def[]
 --
 
-[[glossary-api-key]] API key ::
-// tag::api-key-def[]
-A unique identifier that you can use for authentication when submitting {es} requests. 
-When TLS is enabled, all requests must be authenticated using either basic authentication 
-(user name and password) or an API key.
-// end::api-key-def[]
-
-
-[[glossary-auto-follow-pattern]] auto-follow pattern ::
-// tag::auto-follow-pattern-def[]
-An <<glossary-index-pattern,index pattern>> that automatically configures new indices as 
-<<glossary-follower-index,follower indices>> for <<glossary-ccr,{ccr}>>.
-For more information, see {ref}/ccr-auto-follow.html#_managing_auto_follow_patterns[Managing auto follow patterns].
-// end::auto-follow-pattern-def[]
-
 [[glossary-cluster]] cluster ::
 // tag::cluster-def[]
-One or more <<glossary-node,nodes>> that share the
-same cluster name. Each cluster has a single master node, which is
-chosen automatically by the cluster and can be replaced if it fails.
+A cluster consists of one or more <<glossary-node,nodes>> which share the
+same cluster name. Each cluster has a single master node which is
+chosen automatically by the cluster and which can be replaced if the
+current master node fails.
 // end::cluster-def[]
 
-[[glossary-cold-phase]] cold phase ::
-// tag::cold-phase-def[]
-The third possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
-In the cold phase, an index is no longer updated and seldom queried. 
-The information still needs to be searchable, but it’s okay if those queries are slower.
-// end::cold-phase-def[]
-
 [[glossary-ccr]] {ccr} (CCR)::
 // tag::ccr-def[]
-A feature that enables you to replicate indices in remote clusters to your
+The {ccr} feature enables you to replicate indices in remote clusters to your
 local cluster. For more information, see 
 {ref}/xpack-ccr.html[{ccr-cap}].  
 // end::ccr-def[]
   
 [[glossary-ccs]] {ccs} (CCS)::
 // tag::ccs-def[]
-A feature that enables any node to act as a federated client across
+The {ccs} feature enables any node to act as a federated client across
 multiple clusters.
 See {ref}/modules-cross-cluster-search.html[Search across clusters].
 // end::ccs-def[]
 
-[[glossary-delete-phase]] delete phase ::
-// tag::delete-phase-def[]
-The last possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
-In the delete phase, an index is no longer needed and can safely be deleted.
-// end::delete-phase-def[]
-
 [[glossary-document]] document ::
 +
 --
@@ -120,22 +93,10 @@ hence it is called a filter. Filters are simple checks for set inclusion or excl
 In most cases, the goal of filtering is to reduce the number of documents that have to be examined.
 // end::filter-def[]
 
-[[glossary-flush]] flush ::
-// tag::flush-def[]
-Peform a Lucene commit to write index updates in the transaction log (translog) to disk. 
-Because a Lucene commit is a relatively expensive operation, 
-{es} records index and delete operations in the translog and 
-automatically flushes changes to disk in batches.
-To recover from a crash, operations that have been acknowledged but not yet committed 
-can be replayed from the translog.
-Before upgrading, you can explicitly call the {ref}/indices-flush.html[Flush] API
-to ensure that all changes are committed to disk.
-// end::flush-def[]
-
 [[glossary-follower-index]] follower index ::  
 // tag::follower-index-def[]
-The target index for <<glossary-ccr,{ccr}>>. A follower index exists
-in a local cluster and replicates a <<glossary-leader-index,leader index>>.
+Follower indices are the target indices for <<glossary-ccr,{ccr}>>. They exist
+in your local cluster and replicate <<glossary-leader-index,leader indices>>.
 // end::follower-index-def[]
 
 [[glossary-force-merge]] force merge ::  
@@ -146,7 +107,7 @@ and free up the space used by deleted documents.
 // end::force-merge-def-short[]
 You should not force merge indices that are actively being written to.
 Merging is normally performed automatically, but you can use force merge after 
-<<glossary-rollover,rollover>> to reduce the shards in the old index to a single segment. 
+<<glossary-rollover, rollover>> to reduce the shards in the old index to a single segment. 
 See the {ref}/indices-forcemerge.html[force merge API].
 // end::force-merge-def[]
 
@@ -162,19 +123,6 @@ before you are ready to archive or delete them.
 See the {ref}/freeze-index-api.html[freeze API].
 // end::freeze-def[]
 
-[[glossary-frozen-index]] frozen index ::  
-// tag::frozen-index-def[]
-An index reduced to a low overhead state that still enables occasional searches.
-Frozen indices use a memory-efficient shard implementation and throttle searches to conserve resources.
-Searching a frozen index is lower overhead than re-opening a closed index to enable searching.
-// end::frozen-index-def[]
-
-[[glossary-hot-phase]] hot phase ::
-// tag::hot-phase-def[]
-The first possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
-In the hot phase, an index is actively updated and queried. 
-// end::hot-phase-def[]
-
 [[glossary-id]] id ::
 // tag::id-def[]
 The ID of a <<glossary-document,document>> identifies a document. The
@@ -186,12 +134,11 @@ then it will be auto-generated. (also see <<glossary-routing,routing>>)
 +
 --
 // tag::index-def[]
-// tag::index-def-short[]
-An optimized collection of JSON documents. Each document is a collection of fields, 
-the key-value pairs that contain your data. 
-// end::index-def-short[]
+An index is like a _table_ in a relational database. It has a
+<<glossary-mapping,mapping>> which contains a <<glossary-type,type>>,
+which contains the <<glossary-field,field>> in the index.
 
-An index is a logical namespace that maps to one or more
+An index is a logical namespace which maps to one or more
 <<glossary-primary-shard,primary shards>> and can have zero or more
 <<glossary-replica-shard,replica shards>>.
 // end::index-def[]
@@ -202,38 +149,17 @@ An index is a logical namespace that maps to one or more
 --
 // tag::index-alias-def[]
 // tag::index-alias-desc[]
-An index alias is a logical name used to reference one or more indices.
+An index alias is a secondary name
+used to refer to one or more existing indices.
 
-Most {es} APIs accept an index alias in place of an index name.
+Most {es} APIs accept an index alias
+in place of an index name.
 // end::index-alias-desc[]
 
 See {ref}/indices-add-alias.html[Add index alias].
 // end::index-alias-def[]
 --
 
-[[glossary-index-lifecycle]] index lifecycle ::
-// tag::index-lifecycle-def[]
-The four phases an index can transition through: 
-<<glossary-hot-phase,hot>>, <<glossary-warm-phase,warm>>,
-<<glossary-cold-phase,cold>>, and <<glossary-delete-phase,delete>>.
-For more information, see {ref}/ilm-policy-definition.html[Index lifecycle].
-// end::index-lifecycle-def[]
-
-[[glossary-index-lifecycle-policy]] index lifecycle policy ::
-// tag::index-lifecycle-policy-def[]
-Specifies how an index moves between phases in the index lifecycle and 
-what actions to perform during each phase. 
-// end::index-lifecycle-policy-def[]
-
-[[glossary-index-pattern]] index pattern ::
-// tag::index-pattern-def[]
-A string that can contain the `*` wildcard to match multiple index names.
-In most cases, the index parameter in an {es} request can be the name of a specific index, 
-a list of index names, or an index pattern.
-For example, if you have the indices `datastream-000001`, `datastream-000002`, and `datastream-000003`,
-to search across all three you could use the `datastream-*` index pattern.  
-// end::index-pattern-def[]
-
 [[glossary-index-template]] index template ::
 +
 --
@@ -241,24 +167,18 @@ to search across all three you could use the `datastream-*` index pattern.
 // tag::index-template-def-short[]
 Defines settings and mappings to apply to new indexes that match a simple naming pattern, such as _logs-*_. 
 // end::index-template-def-short[]
-
 An index template can also attach a lifecycle policy to the new index.
-Index templates are used to automatically configure indices created during <<glossary-rollover,rollover>>. 
+Index templates are used to automatically configure indices created during <<glossary-rollover, rollover>>. 
 // end::index-template-def[]
 --
 
 [[glossary-leader-index]] leader index ::  
 // tag::leader-index-def[]
-The source index for <<glossary-ccr,{ccr}>>. A leader index exists
-on a remote cluster and is replicated to 
+Leader indices are the source indices for <<glossary-ccr,{ccr}>>. They exist
+on remote clusters and are replicated to 
 <<glossary-follower-index,follower indices>>.
 // end::leader-index-def[]
 
-[[glossary-local-cluster]] local cluster ::
-// tag::local-cluster-def[]
-The cluster that pulls data from a <<glossary-remote-cluster,remote cluster>> in {ccs} or {ccr}.
-// end::local-cluster-def[]
-
 [[glossary-mapping]] mapping ::
 +
 --
@@ -274,12 +194,18 @@ automatically when a document is indexed.
 --
 
 [[glossary-node]] node ::
++
+--
 // tag::node-def[]
-A running instance of {es} that belongs to a
+A node is a running instance of Elasticsearch which belongs to a
 <<glossary-cluster,cluster>>. Multiple nodes can be started on a single
 server for testing purposes, but usually you should have one node per
 server.
+
+At startup, a node will use unicast to discover an existing cluster with
+the same cluster name and will try to join that cluster.
 // end::node-def[]
+--
 
 [[glossary-primary-shard]] primary shard ::
 +
@@ -341,18 +267,9 @@ during the following processes:
 [[glossary-reindex]] reindex ::
 
 // tag::reindex-def[]
-To cycle through some or all documents in one or more indices, re-writing them into the same 
-or new index in a local or remote cluster. This is most commonly done to update mappings, or to upgrade {es} between two incompatible index versions.
+To cycle through some or all documents in one or more indices, re-writing them into the same or new index in a local or remote cluster. This is most commonly done to update mappings, or to upgrade Elasticsearch between two incompatible index versions.
 // end::reindex-def[]
 
-[[glossary-remote-cluster]] remote cluster ::
-
-// tag::remote-cluster-def[]
-A separate cluster, often in a different data center or locale, that contains indices that 
-can be replicated or searched by the <<glossary-local-cluster,local cluster>>. 
-The connection to a remote cluster is unidirectional.
-// end::remote-cluster-def[]
-
 [[glossary-replica-shard]] replica shard ::
 +
 --
@@ -377,35 +294,14 @@ shard will never be started on the same node as its primary shard.
 --
 // tag::rollover-def[]
 // tag::rollover-def-short[]
-Redirect an index alias to begin writing to a new index when the existing index reaches 
-a certain size, number of docs, or age. 
+Redirect an alias to begin writing to a new index when the existing index reaches a certain age, number of docs, or size. 
 // end::rollover-def-short[]
-
-The new index is automatically configured according to any matching <<glossary-index-template,index templates>>. 
+The new index is automatically configured according to any matching <<glossary-index-template, index templates>>. 
 For example, if you're indexing log data, you might use rollover to create daily or weekly indices. 
 See the {ref}/indices-rollover-index.html[rollover index API].
 // end::rollover-def[]
 --
 
-[[glossary-rollup]] rollup ::
-// tag::rollup-def[]
-Summarize high-granularity data into a more compressed format to 
-maintain access to historical data in a cost-effective way.
-// end::rollup-def[]
-
-[[glossary-rollup-index]] rollup index ::
-// tag::rollup-index-def[]
-A special type of index for storing historical data at reduced granularity. 
-Documents are summarized and indexed into a rollup index by a <<glossary-rollup-job,rollup job>>.
-// end::rollup-index-def[]
-
-[[glossary-rollup-job]] rollup job ::
-// tag::rollup-job-def[]
-A background task that runs continuously to summarize documents in an index and 
-index the summaries into a separate rollup index. 
-The job configuration controls what information is rolled up and how often. 
-// end::rollup-job-def[]
-
 [[glossary-routing]] routing ::
 +
 --
@@ -444,36 +340,14 @@ nodes.
 --
 
 [[glossary-shrink]] shrink ::
-+
---
 // tag::shrink-def[]
 // tag::shrink-def-short[]
 Reduce the number of primary shards in an index.
 // end::shrink-def-short[]
-
 You can shrink an index to reduce its overhead when the request volume drops.
 For example, you might opt to shrink an index once it is no longer the write index. 
 See the {ref}/indices-shrink-index.html[shrink index API].
 // end::shrink-def[]
---
-
-[[glossary-snapshot]] snapshot ::
-// tag::snapshot-def[]
-A backup taken from a running {es} cluster. 
-You can take snapshots of individual indices or of the entire cluster. 
-// end::snapshot-def[]
-
-[[glossary-snapshot-lifecycle-policy]] snapshot lifecycle policy ::
-// tag::snapshot-lifecycle-policy-def[]
-Specifies how frequently to perform automatic backups of a cluster and 
-how long to retain the resulting snapshots. 
-// end::snapshot-lifecycle-policy-def[]
-
-[[glossary-snapshot-repository]] snapshot repository ::
-// tag::snapshot-repository-def[]
-Specifies where snapshots are to be stored. 
-Snapshots can be written to a shared filesystem or to a remote repository.
-// end::snapshot-repository-def[]
 
 [[glossary-source_field]] source field ::
 // tag::source-field-def[]
@@ -525,9 +399,3 @@ A type used to represent the _type_ of document, e.g. an `email`, a `user`, or a
 Types are deprecated and are in the process of being removed.
 See {ref}/removal-of-types.html[Removal of mapping types].
 // end::type-def[]
-
-[[glossary-warm-phase]] warm phase ::
-// tag::warm-phase-def[]
-The second possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
-In the warm phase, an index is generally optimized for search and no longer updated.
-// end::warm-phase-def[]
diff --git a/docs/reference/ilm/policy-definitions.asciidoc b/docs/reference/ilm/policy-definitions.asciidoc
index 1c917eade49d2..d66a53ecea55a 100644
--- a/docs/reference/ilm/policy-definitions.asciidoc
+++ b/docs/reference/ilm/policy-definitions.asciidoc
@@ -113,7 +113,7 @@ policy definition.
   - <<ilm-allocate-action,Allocate>>
   - <<ilm-freeze-action,Freeze>>
 * Delete
-  - <<ilm-wait-for-snapshot-action,Wait For Snapshot>>
+  - <<ilm-delete-action,Wait For Snapshot>>
   - <<ilm-delete-action,Delete>>
 
 [[ilm-allocate-action]]
diff --git a/docs/reference/images/analysis/token-graph-basic.svg b/docs/reference/images/analysis/token-graph-basic.svg
deleted file mode 100644
index 99c2b0cb24f5c..0000000000000
--- a/docs/reference/images/analysis/token-graph-basic.svg
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="602px" height="54px" viewBox="0 0 602 54" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
-    <title>Slice 1</title>
-    <desc>Created with Sketch.</desc>
-    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-        <g id="token-graph-basic" transform="translate(1.000000, 1.000000)">
-            <g id="Group">
-                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
-                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
-                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
-                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
-                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
-                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="73.96063 10.47244 152.35432 10.47244 152.35432 41.842511 73.96063 41.842511"></polygon>
-                <path d="M84.39813,33.07748 L84.39813,19.718097 L89.445005,19.718097 C90.3304217,19.718097 91.007505,19.7597637 91.476255,19.843097 C92.1220883,19.9576803 92.6689633,20.171222 93.11688,20.483722 C93.5647967,20.7858053 93.9241717,21.2128887 94.195005,21.764972 C94.4658383,22.3170553 94.601255,22.9264303 94.601255,23.593097 C94.601255,24.7285137 94.2366717,25.6920553 93.507505,26.483722 C92.788755,27.264972 91.4814633,27.655597 89.58563,27.655597 L86.163755,27.655597 L86.163755,33.07748 L84.39813,33.07748 L84.39813,33.07748 Z M86.163755,26.0774724 L89.61688,26.0774724 C90.7627133,26.0774724 91.5752133,25.8639307 92.05438,25.4368474 C92.5335467,25.0097641 92.77313,24.4108057 92.77313,23.6399724 C92.77313,23.0774724 92.632505,22.5983057 92.351255,22.2024724 C92.070005,21.8066391 91.695005,21.5462224 91.226255,21.4212224 C90.9345883,21.3378891 90.3877133,21.2962224 89.58563,21.2962224 L86.163755,21.2962224 L86.163755,26.0774724 L86.163755,26.0774724 Z M96.029203,28.23373 C96.029203,26.4420583 96.5239947,25.1191391 97.513578,24.2649724 C98.3469113,23.5462224 99.3625363,23.1868474 100.560453,23.1868474 C101.893786,23.1868474 102.982328,23.6243474 103.826078,24.4993474 C104.669828,25.3639307 105.091703,26.5670583 105.091703,28.10873 C105.091703,29.3483133 104.904203,30.32748 104.529203,31.04623 C104.154203,31.7545633 103.607328,32.3066467 102.888578,32.70248 C102.180245,33.0983133 101.404203,33.29623 100.560453,33.29623 C99.2062863,33.29623 98.1125363,32.8639383 97.279203,31.999355 C96.4458697,31.124355 96.029203,29.8691467 96.029203,28.23373 Z M97.716703,28.23373 C97.716703,29.4733133 97.982328,30.405605 98.513578,31.030605 C99.0552447,31.6451883 99.7375363,31.95248 100.560453,31.95248 C101.372953,31.95248 102.050036,31.6451883 102.591703,31.030605 C103.13337,30.405605 103.404203,29.4576883 103.404203,28.186855 C103.404203,26.9889333 103.13337,26.0826807 102.591703,25.4680974 C102.050036,24.8535141 101.372953,24.5462224 100.560453,24.5462224 C99.7375363,24.5462224 99.0552447,24.8535141 98.513578,25.4680974 C97.982328,26.0722641 97.716703,26.9941416 97.716703,28.23373 Z M108.795545,33.07748 L105.826795,23.405597 L107.52992,23.405597 L109.06117,28.9837296 L109.639295,31.0618546 C109.670545,30.9576879 109.837212,30.2910213 110.139295,29.0618546 L111.68617,23.405597 L113.37367,23.405597 L114.81117,29.0149796 L115.295545,30.8587296 L115.858045,28.9993546 L117.514295,23.405597 L119.108045,23.405597 L116.076795,33.07748 L114.37367,33.07748 L112.84242,27.2805974 L112.46742,25.6399724 L110.514295,33.07748 L108.795545,33.07748 Z M127.112241,29.968105 L128.799741,30.17123 C128.528908,31.1608133 128.034116,31.9316467 127.315366,32.48373 C126.596616,33.0253967 125.674741,33.29623 124.549741,33.29623 C123.133074,33.29623 122.008074,32.8639383 121.174741,31.999355 C120.351824,31.124355 119.940366,29.9003967 119.940366,28.32748 C119.940366,26.6920583 120.357033,25.4264307 121.190366,24.5305974 C122.034116,23.6347641 123.122658,23.1868474 124.455991,23.1868474 C125.747658,23.1868474 126.799741,23.6295557 127.612241,24.5149724 C128.435158,25.3899724 128.846616,26.6243499 128.846616,28.218105 C128.846616,28.3222717 128.846616,28.468105 128.846616,28.655605 L121.627866,28.655605 C121.690366,29.718105 121.992449,30.5358133 122.534116,31.10873 C123.075783,31.67123 123.747658,31.95248 124.549741,31.95248 C125.153908,31.95248 125.669533,31.79623 126.096616,31.48373 C126.523699,31.1608133 126.862241,30.655605 127.112241,29.968105 Z M121.721616,27.3118474 L127.127866,27.3118474 C127.054949,26.4993474 126.846616,25.8899724 126.502866,25.4837224 C125.982033,24.8483057 125.304949,24.5305974 124.471616,24.5305974 C123.721616,24.5305974 123.086199,24.7858057 122.565366,25.2962224 C122.054949,25.7962224 121.773699,26.4680974 121.721616,27.3118474 Z M130.847333,33.07748 L130.847333,23.405597 L132.316083,23.405597 L132.316083,24.874347 C132.691083,24.186847 133.034833,23.733722 133.347333,23.514972 C133.67025,23.296222 134.024416,23.186847 134.409833,23.186847 C134.961916,23.186847 135.524416,23.3639303 136.097333,23.718097 L135.534833,25.233722 C135.128583,24.9941387 134.727541,24.874347 134.331708,24.874347 C133.967125,24.874347 133.644208,24.983722 133.362958,25.202472 C133.081708,25.421222 132.878583,25.718097 132.753583,26.093097 C132.566083,26.6764303 132.472333,27.3170579 132.472333,28.0149796 L132.472333,33.07748 L130.847333,33.07748 L130.847333,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="265.96063 10.47244 332.38583 10.47244 332.38583 41.842511 265.96063 41.842511"></polygon>
-                <path d="M280.82782,28.78061 L282.48407,28.639985 C282.567403,29.3066517 282.749695,29.8535267 283.030945,30.28061 C283.322612,30.7076933 283.770528,31.0566517 284.374695,31.327485 C284.989278,31.5879017 285.676778,31.71811 286.437195,31.71811 C287.103862,31.71811 287.697612,31.6191517 288.218445,31.421235 C288.739278,31.2233183 289.124695,30.952485 289.374695,30.608735 C289.624695,30.2545683 289.749695,29.8691517 289.749695,29.452485 C289.749695,29.0358183 289.624695,28.671235 289.374695,28.358735 C289.135112,28.0462299 288.739278,27.7806024 288.187195,27.5618524 C287.822612,27.4264357 287.025737,27.2128941 285.79657,26.9212274 C284.57782,26.6191441 283.723653,26.3378941 283.23407,26.0774774 C282.588237,25.7441441 282.10907,25.3326857 281.79657,24.8431024 C281.48407,24.3431024 281.32782,23.7858107 281.32782,23.1712274 C281.32782,22.5045607 281.520528,21.8795607 281.905945,21.2962274 C282.291362,20.7024774 282.848653,20.2545607 283.57782,19.9524774 C284.317403,19.6503941 285.135112,19.4993524 286.030945,19.4993524 C287.020528,19.4993524 287.89032,19.6608107 288.64032,19.9837274 C289.400737,20.2962274 289.98407,20.7649774 290.39032,21.3899774 C290.79657,22.0045607 291.01532,22.7024774 291.04657,23.4837274 L289.35907,23.6087274 C289.26532,22.7649774 288.95282,22.1295607 288.42157,21.7024774 C287.900737,21.2649774 287.129903,21.0462274 286.10907,21.0462274 C285.036153,21.0462274 284.254903,21.2441441 283.76532,21.6399774 C283.275737,22.0358107 283.030945,22.5097691 283.030945,23.0618524 C283.030945,23.5410191 283.208028,23.9316441 283.562195,24.2337274 C283.895528,24.5462274 284.780945,24.8691441 286.218445,25.2024774 C287.655945,25.5253941 288.64032,25.8066441 289.17157,26.0462274 C289.95282,26.4003941 290.525737,26.8535191 290.89032,27.4056024 C291.26532,27.9576908 291.45282,28.59311 291.45282,29.31186 C291.45282,30.0201933 291.249695,30.6920683 290.843445,31.327485 C290.437195,31.952485 289.853862,32.4420683 289.093445,32.796235 C288.333028,33.139985 287.473653,33.31186 286.51532,33.31186 C285.306987,33.31186 284.291362,33.1347767 283.468445,32.78061 C282.655945,32.4264433 282.01532,31.8951933 281.54657,31.18686 C281.088237,30.4785267 280.848653,29.6764433 280.82782,28.78061 Z M293.662018,33.077485 L293.662018,19.718102 L295.302643,19.718102 L295.302643,24.514977 C296.06306,23.6295603 297.026601,23.186852 298.193268,23.186852 C298.912018,23.186852 299.53181,23.327477 300.052643,23.608727 C300.583893,23.889977 300.964101,24.280602 301.193268,24.780602 C301.422435,25.280602 301.537018,26.0045603 301.537018,26.952477 L301.537018,33.077485 L299.896393,33.077485 L299.896393,26.952477 C299.896393,26.1295603 299.71931,25.530602 299.365143,25.155602 C299.010976,24.780602 298.505768,24.593102 297.849518,24.593102 C297.370351,24.593102 296.917226,24.7233103 296.490143,24.983727 C296.06306,25.233727 295.755768,25.5722687 295.568268,25.999352 C295.391185,26.4264353 295.302643,27.0201853 295.302643,27.780602 L295.302643,33.077485 L293.662018,33.077485 L293.662018,33.077485 Z M303.428375,28.233735 C303.428375,26.4420633 303.923167,25.1191441 304.91275,24.2649774 C305.746083,23.5462274 306.761708,23.1868524 307.959625,23.1868524 C309.292958,23.1868524 310.3815,23.6243524 311.22525,24.4993524 C312.069,25.3639357 312.490875,26.5670633 312.490875,28.108735 C312.490875,29.3483183 312.303375,30.327485 311.928375,31.046235 C311.553375,31.7545683 311.0065,32.3066517 310.28775,32.702485 C309.579417,33.0983183 308.803375,33.296235 307.959625,33.296235 C306.605458,33.296235 305.511708,32.8639433 304.678375,31.99936 C303.845042,31.12436 303.428375,29.8691517 303.428375,28.233735 Z M305.115875,28.233735 C305.115875,29.4733183 305.3815,30.40561 305.91275,31.03061 C306.454417,31.6451933 307.136708,31.952485 307.959625,31.952485 C308.772125,31.952485 309.449208,31.6451933 309.990875,31.03061 C310.532542,30.40561 310.803375,29.4576933 310.803375,28.18686 C310.803375,26.9889383 310.532542,26.0826857 309.990875,25.4681024 C309.449208,24.8535191 308.772125,24.5462274 307.959625,24.5462274 C307.136708,24.5462274 306.454417,24.8535191 305.91275,25.4681024 C305.3815,26.0722691 305.115875,26.9941466 305.115875,28.233735 Z M317.991577,31.608735 L318.225952,33.06186 C317.767619,33.15561 317.35616,33.202485 316.991577,33.202485 C316.397827,33.202485 315.934285,33.108735 315.600952,32.921235 C315.278035,32.733735 315.048869,32.4889433 314.913452,32.18686 C314.778035,31.87436 314.710327,31.2285267 314.710327,30.24936 L314.710327,24.6712274 L313.507202,24.6712274 L313.507202,23.4056024 L314.710327,23.4056024 L314.710327,21.0149774 L316.335327,20.0306024 L316.335327,23.4056024 L317.991577,23.4056024 L317.991577,24.6712274 L316.335327,24.6712274 L316.335327,30.34311 C316.335327,30.8014433 316.361369,31.0983183 316.413452,31.233735 C316.475952,31.3691517 316.569702,31.4785267 316.694702,31.56186 C316.830119,31.6347767 317.022827,31.671235 317.272827,31.671235 C317.44991,31.671235 317.689494,31.6504017 317.991577,31.608735 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="447.53366 10.47244 518.43127 10.47244 518.43127 41.842511 447.53366 41.842511"></polygon>
-                <path d="M471.62164,31.49936 L471.62164,33.077485 L462.793545,33.077485 C462.783129,32.6816517 462.845629,32.3014433 462.981045,31.93686 C463.210212,31.3326933 463.569587,30.7389433 464.05917,30.15561 C464.55917,29.5722767 465.27792,28.9004017 466.21542,28.139985 C467.663337,26.9524799 468.642504,26.0097691 469.15292,25.3118524 C469.663337,24.6139357 469.918545,23.9576857 469.918545,23.3431024 C469.918545,22.6868524 469.68417,22.1399774 469.21542,21.7024774 C468.757087,21.2545607 468.15292,21.0306024 467.40292,21.0306024 C466.611254,21.0306024 465.975837,21.2701857 465.49667,21.7493524 C465.02792,22.2181024 464.793545,22.8691441 464.793545,23.7024774 L463.106045,23.5306024 C463.220629,22.2701857 463.65292,21.3118524 464.40292,20.6556024 C465.163337,19.9993524 466.173754,19.6712274 467.43417,19.6712274 C468.71542,19.6712274 469.731045,20.0253941 470.481045,20.7337274 C471.231025,21.4420607 471.606015,22.3222691 471.606015,23.3743524 C471.606015,23.9056024 471.49664,24.4316441 471.27789,24.9524774 C471.05916,25.4733107 470.694587,26.0201857 470.18417,26.5931024 C469.68417,27.1556024 468.84042,27.9368549 467.65292,28.93686 C466.673754,29.7597767 466.043545,30.3222767 465.762295,30.62436 C465.481045,30.9160267 465.251879,31.2076933 465.074795,31.49936 L471.62164,31.49936 Z M473.387997,26.4837274 C473.387997,24.9108107 473.549456,23.6451857 473.872372,22.6868524 C474.195289,21.7181024 474.674456,20.9733107 475.309872,20.4524774 C475.955706,19.9316441 476.762997,19.6712274 477.731747,19.6712274 C478.450497,19.6712274 479.080706,19.8170607 479.622372,20.1087274 C480.164039,20.3899774 480.606747,20.8066441 480.950497,21.3587274 C481.304664,21.9003941 481.580706,22.5618524 481.778622,23.3431024 C481.986956,24.1139357 482.091122,25.1608107 482.091122,26.4837274 C482.091122,28.0566491 481.929664,29.327485 481.606747,30.296235 C481.283831,31.2545683 480.799456,31.99936 480.153622,32.53061 C479.518206,33.0514433 478.710914,33.31186 477.731747,33.31186 C476.450497,33.31186 475.440081,32.8483183 474.700497,31.921235 C473.825497,30.8066517 473.387997,28.9941491 473.387997,26.4837274 Z M475.059872,26.4837274 C475.059872,28.6816491 475.315081,30.1451933 475.825497,30.87436 C476.346331,31.59311 476.981747,31.952485 477.731747,31.952485 C478.492164,31.952485 479.127581,31.5879017 479.637997,30.858735 C480.148414,30.1295683 480.403622,28.6712325 480.403622,26.4837274 C480.403622,24.2858107 480.148414,22.8274774 479.637997,22.1087274 C479.127581,21.3899774 478.486956,21.0306024 477.716122,21.0306024 C476.966122,21.0306024 476.367164,21.3483107 475.919247,21.9837274 C475.346331,22.7962274 475.059872,24.2962274 475.059872,26.4837274 Z M483.763729,26.4837274 C483.763729,24.9108107 483.925188,23.6451857 484.248104,22.6868524 C484.571021,21.7181024 485.050188,20.9733107 485.685604,20.4524774 C486.331438,19.9316441 487.138729,19.6712274 488.107479,19.6712274 C488.826229,19.6712274 489.456438,19.8170607 489.998104,20.1087274 C490.539771,20.3899774 490.982479,20.8066441 491.326229,21.3587274 C491.680396,21.9003941 491.956438,22.5618524 492.154354,23.3431024 C492.362688,24.1139357 492.466854,25.1608107 492.466854,26.4837274 C492.466854,28.0566491 492.305396,29.327485 491.982479,30.296235 C491.659563,31.2545683 491.175188,31.99936 490.529354,32.53061 C489.893938,33.0514433 489.086646,33.31186 488.107479,33.31186 C486.826229,33.31186 485.815813,32.8483183 485.076229,31.921235 C484.201229,30.8066517 483.763729,28.9941491 483.763729,26.4837274 Z M485.435604,26.4837274 C485.435604,28.6816491 485.690813,30.1451933 486.201229,30.87436 C486.722063,31.59311 487.357479,31.952485 488.107479,31.952485 C488.867896,31.952485 489.503313,31.5879017 490.013729,30.858735 C490.524146,30.1295683 490.779354,28.6712325 490.779354,26.4837274 C490.779354,24.2858107 490.524146,22.8274774 490.013729,22.1087274 C489.503313,21.3899774 488.862688,21.0306024 488.091854,21.0306024 C487.341854,21.0306024 486.742896,21.3483107 486.294979,21.9837274 C485.722063,22.7962274 485.435604,24.2962274 485.435604,26.4837274 Z M494.139461,26.4837274 C494.139461,24.9108107 494.30092,23.6451857 494.623836,22.6868524 C494.946753,21.7181024 495.42592,20.9733107 496.061336,20.4524774 C496.70717,19.9316441 497.514461,19.6712274 498.483211,19.6712274 C499.201961,19.6712274 499.83217,19.8170607 500.373836,20.1087274 C500.915503,20.3899774 501.358211,20.8066441 501.701961,21.3587274 C502.056128,21.9003941 502.33217,22.5618524 502.530086,23.3431024 C502.73842,24.1139357 502.842586,25.1608107 502.842586,26.4837274 C502.842586,28.0566491 502.681128,29.327485 502.358211,30.296235 C502.035295,31.2545683 501.55092,31.99936 500.905086,32.53061 C500.26967,33.0514433 499.462378,33.31186 498.483211,33.31186 C497.201961,33.31186 496.191545,32.8483183 495.451961,31.921235 C494.576961,30.8066517 494.139461,28.9941491 494.139461,26.4837274 Z M495.811336,26.4837274 C495.811336,28.6816491 496.066545,30.1451933 496.576961,30.87436 C497.097795,31.59311 497.733211,31.952485 498.483211,31.952485 C499.243628,31.952485 499.879045,31.5879017 500.389461,30.858735 C500.899878,30.1295683 501.155086,28.6712325 501.155086,26.4837274 C501.155086,24.2858107 500.899878,22.8274774 500.389461,22.1087274 C499.879045,21.3899774 499.23842,21.0306024 498.467586,21.0306024 C497.717586,21.0306024 497.118628,21.3483107 496.670711,21.9837274 C496.097795,22.7962274 495.811336,24.2962274 495.811336,26.4837274 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-            </g>
-        </g>
-    </g>
-</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-wd.svg b/docs/reference/images/analysis/token-graph-wd.svg
deleted file mode 100644
index cdbbfb8a0845c..0000000000000
--- a/docs/reference/images/analysis/token-graph-wd.svg
+++ /dev/null
@@ -1,52 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="602px" height="123px" viewBox="0 0 602 123" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
-    <title>Slice 1</title>
-    <desc>Created with Sketch.</desc>
-    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-        <g id="Analysis-(1)" transform="translate(1.000000, 1.000000)">
-            <g id="Group">
-                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
-                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
-                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
-                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
-                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
-                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="15.088913 39.14516 34.52198 39.14516 34.52198 60.971927 15.088913 60.971927"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="199.08891 39.14516 218.52197 39.14516 218.52197 60.971927 199.08891 60.971927"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="73.96063 10.47244 152.35432 10.47244 152.35432 41.842511 73.96063 41.842511"></polygon>
-                <path d="M84.39813,33.07748 L84.39813,19.718097 L89.445005,19.718097 C90.3304217,19.718097 91.007505,19.7597637 91.476255,19.843097 C92.1220883,19.9576803 92.6689633,20.171222 93.11688,20.483722 C93.5647967,20.7858053 93.9241717,21.2128887 94.195005,21.764972 C94.4658383,22.3170553 94.601255,22.9264303 94.601255,23.593097 C94.601255,24.7285137 94.2366717,25.6920553 93.507505,26.483722 C92.788755,27.264972 91.4814633,27.655597 89.58563,27.655597 L86.163755,27.655597 L86.163755,33.07748 L84.39813,33.07748 L84.39813,33.07748 Z M86.163755,26.0774724 L89.61688,26.0774724 C90.7627133,26.0774724 91.5752133,25.8639307 92.05438,25.4368474 C92.5335467,25.0097641 92.77313,24.4108057 92.77313,23.6399724 C92.77313,23.0774724 92.632505,22.5983057 92.351255,22.2024724 C92.070005,21.8066391 91.695005,21.5462224 91.226255,21.4212224 C90.9345883,21.3378891 90.3877133,21.2962224 89.58563,21.2962224 L86.163755,21.2962224 L86.163755,26.0774724 L86.163755,26.0774724 Z M96.029203,28.23373 C96.029203,26.4420583 96.5239947,25.1191391 97.513578,24.2649724 C98.3469113,23.5462224 99.3625363,23.1868474 100.560453,23.1868474 C101.893786,23.1868474 102.982328,23.6243474 103.826078,24.4993474 C104.669828,25.3639307 105.091703,26.5670583 105.091703,28.10873 C105.091703,29.3483133 104.904203,30.32748 104.529203,31.04623 C104.154203,31.7545633 103.607328,32.3066467 102.888578,32.70248 C102.180245,33.0983133 101.404203,33.29623 100.560453,33.29623 C99.2062863,33.29623 98.1125363,32.8639383 97.279203,31.999355 C96.4458697,31.124355 96.029203,29.8691467 96.029203,28.23373 Z M97.716703,28.23373 C97.716703,29.4733133 97.982328,30.405605 98.513578,31.030605 C99.0552447,31.6451883 99.7375363,31.95248 100.560453,31.95248 C101.372953,31.95248 102.050036,31.6451883 102.591703,31.030605 C103.13337,30.405605 103.404203,29.4576883 103.404203,28.186855 C103.404203,26.9889333 103.13337,26.0826807 102.591703,25.4680974 C102.050036,24.8535141 101.372953,24.5462224 100.560453,24.5462224 C99.7375363,24.5462224 99.0552447,24.8535141 98.513578,25.4680974 C97.982328,26.0722641 97.716703,26.9941416 97.716703,28.23373 Z M108.795545,33.07748 L105.826795,23.405597 L107.52992,23.405597 L109.06117,28.9837296 L109.639295,31.0618546 C109.670545,30.9576879 109.837212,30.2910213 110.139295,29.0618546 L111.68617,23.405597 L113.37367,23.405597 L114.81117,29.0149796 L115.295545,30.8587296 L115.858045,28.9993546 L117.514295,23.405597 L119.108045,23.405597 L116.076795,33.07748 L114.37367,33.07748 L112.84242,27.2805974 L112.46742,25.6399724 L110.514295,33.07748 L108.795545,33.07748 Z M127.112241,29.968105 L128.799741,30.17123 C128.528908,31.1608133 128.034116,31.9316467 127.315366,32.48373 C126.596616,33.0253967 125.674741,33.29623 124.549741,33.29623 C123.133074,33.29623 122.008074,32.8639383 121.174741,31.999355 C120.351824,31.124355 119.940366,29.9003967 119.940366,28.32748 C119.940366,26.6920583 120.357033,25.4264307 121.190366,24.5305974 C122.034116,23.6347641 123.122658,23.1868474 124.455991,23.1868474 C125.747658,23.1868474 126.799741,23.6295557 127.612241,24.5149724 C128.435158,25.3899724 128.846616,26.6243499 128.846616,28.218105 C128.846616,28.3222717 128.846616,28.468105 128.846616,28.655605 L121.627866,28.655605 C121.690366,29.718105 121.992449,30.5358133 122.534116,31.10873 C123.075783,31.67123 123.747658,31.95248 124.549741,31.95248 C125.153908,31.95248 125.669533,31.79623 126.096616,31.48373 C126.523699,31.1608133 126.862241,30.655605 127.112241,29.968105 Z M121.721616,27.3118474 L127.127866,27.3118474 C127.054949,26.4993474 126.846616,25.8899724 126.502866,25.4837224 C125.982033,24.8483057 125.304949,24.5305974 124.471616,24.5305974 C123.721616,24.5305974 123.086199,24.7858057 122.565366,25.2962224 C122.054949,25.7962224 121.773699,26.4680974 121.721616,27.3118474 Z M130.847333,33.07748 L130.847333,23.405597 L132.316083,23.405597 L132.316083,24.874347 C132.691083,24.186847 133.034833,23.733722 133.347333,23.514972 C133.67025,23.296222 134.024416,23.186847 134.409833,23.186847 C134.961916,23.186847 135.524416,23.3639303 136.097333,23.718097 L135.534833,25.233722 C135.128583,24.9941387 134.727541,24.874347 134.331708,24.874347 C133.967125,24.874347 133.644208,24.983722 133.362958,25.202472 C133.081708,25.421222 132.878583,25.718097 132.753583,26.093097 C132.566083,26.6764303 132.472333,27.3170579 132.472333,28.0149796 L132.472333,33.07748 L130.847333,33.07748 L130.847333,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="265.96063 10.47244 332.38583 10.47244 332.38583 41.842511 265.96063 41.842511"></polygon>
-                <path d="M280.82782,28.78061 L282.48407,28.639985 C282.567403,29.3066517 282.749695,29.8535267 283.030945,30.28061 C283.322612,30.7076933 283.770528,31.0566517 284.374695,31.327485 C284.989278,31.5879017 285.676778,31.71811 286.437195,31.71811 C287.103862,31.71811 287.697612,31.6191517 288.218445,31.421235 C288.739278,31.2233183 289.124695,30.952485 289.374695,30.608735 C289.624695,30.2545683 289.749695,29.8691517 289.749695,29.452485 C289.749695,29.0358183 289.624695,28.671235 289.374695,28.358735 C289.135112,28.0462299 288.739278,27.7806024 288.187195,27.5618524 C287.822612,27.4264357 287.025737,27.2128941 285.79657,26.9212274 C284.57782,26.6191441 283.723653,26.3378941 283.23407,26.0774774 C282.588237,25.7441441 282.10907,25.3326857 281.79657,24.8431024 C281.48407,24.3431024 281.32782,23.7858107 281.32782,23.1712274 C281.32782,22.5045607 281.520528,21.8795607 281.905945,21.2962274 C282.291362,20.7024774 282.848653,20.2545607 283.57782,19.9524774 C284.317403,19.6503941 285.135112,19.4993524 286.030945,19.4993524 C287.020528,19.4993524 287.89032,19.6608107 288.64032,19.9837274 C289.400737,20.2962274 289.98407,20.7649774 290.39032,21.3899774 C290.79657,22.0045607 291.01532,22.7024774 291.04657,23.4837274 L289.35907,23.6087274 C289.26532,22.7649774 288.95282,22.1295607 288.42157,21.7024774 C287.900737,21.2649774 287.129903,21.0462274 286.10907,21.0462274 C285.036153,21.0462274 284.254903,21.2441441 283.76532,21.6399774 C283.275737,22.0358107 283.030945,22.5097691 283.030945,23.0618524 C283.030945,23.5410191 283.208028,23.9316441 283.562195,24.2337274 C283.895528,24.5462274 284.780945,24.8691441 286.218445,25.2024774 C287.655945,25.5253941 288.64032,25.8066441 289.17157,26.0462274 C289.95282,26.4003941 290.525737,26.8535191 290.89032,27.4056024 C291.26532,27.9576908 291.45282,28.59311 291.45282,29.31186 C291.45282,30.0201933 291.249695,30.6920683 290.843445,31.327485 C290.437195,31.952485 289.853862,32.4420683 289.093445,32.796235 C288.333028,33.139985 287.473653,33.31186 286.51532,33.31186 C285.306987,33.31186 284.291362,33.1347767 283.468445,32.78061 C282.655945,32.4264433 282.01532,31.8951933 281.54657,31.18686 C281.088237,30.4785267 280.848653,29.6764433 280.82782,28.78061 Z M293.662018,33.077485 L293.662018,19.718102 L295.302643,19.718102 L295.302643,24.514977 C296.06306,23.6295603 297.026601,23.186852 298.193268,23.186852 C298.912018,23.186852 299.53181,23.327477 300.052643,23.608727 C300.583893,23.889977 300.964101,24.280602 301.193268,24.780602 C301.422435,25.280602 301.537018,26.0045603 301.537018,26.952477 L301.537018,33.077485 L299.896393,33.077485 L299.896393,26.952477 C299.896393,26.1295603 299.71931,25.530602 299.365143,25.155602 C299.010976,24.780602 298.505768,24.593102 297.849518,24.593102 C297.370351,24.593102 296.917226,24.7233103 296.490143,24.983727 C296.06306,25.233727 295.755768,25.5722687 295.568268,25.999352 C295.391185,26.4264353 295.302643,27.0201853 295.302643,27.780602 L295.302643,33.077485 L293.662018,33.077485 L293.662018,33.077485 Z M303.428375,28.233735 C303.428375,26.4420633 303.923167,25.1191441 304.91275,24.2649774 C305.746083,23.5462274 306.761708,23.1868524 307.959625,23.1868524 C309.292958,23.1868524 310.3815,23.6243524 311.22525,24.4993524 C312.069,25.3639357 312.490875,26.5670633 312.490875,28.108735 C312.490875,29.3483183 312.303375,30.327485 311.928375,31.046235 C311.553375,31.7545683 311.0065,32.3066517 310.28775,32.702485 C309.579417,33.0983183 308.803375,33.296235 307.959625,33.296235 C306.605458,33.296235 305.511708,32.8639433 304.678375,31.99936 C303.845042,31.12436 303.428375,29.8691517 303.428375,28.233735 Z M305.115875,28.233735 C305.115875,29.4733183 305.3815,30.40561 305.91275,31.03061 C306.454417,31.6451933 307.136708,31.952485 307.959625,31.952485 C308.772125,31.952485 309.449208,31.6451933 309.990875,31.03061 C310.532542,30.40561 310.803375,29.4576933 310.803375,28.18686 C310.803375,26.9889383 310.532542,26.0826857 309.990875,25.4681024 C309.449208,24.8535191 308.772125,24.5462274 307.959625,24.5462274 C307.136708,24.5462274 306.454417,24.8535191 305.91275,25.4681024 C305.3815,26.0722691 305.115875,26.9941466 305.115875,28.233735 Z M317.991577,31.608735 L318.225952,33.06186 C317.767619,33.15561 317.35616,33.202485 316.991577,33.202485 C316.397827,33.202485 315.934285,33.108735 315.600952,32.921235 C315.278035,32.733735 315.048869,32.4889433 314.913452,32.18686 C314.778035,31.87436 314.710327,31.2285267 314.710327,30.24936 L314.710327,24.6712274 L313.507202,24.6712274 L313.507202,23.4056024 L314.710327,23.4056024 L314.710327,21.0149774 L316.335327,20.0306024 L316.335327,23.4056024 L317.991577,23.4056024 L317.991577,24.6712274 L316.335327,24.6712274 L316.335327,30.34311 C316.335327,30.8014433 316.361369,31.0983183 316.413452,31.233735 C316.475952,31.3691517 316.569702,31.4785267 316.694702,31.56186 C316.830119,31.6347767 317.022827,31.671235 317.272827,31.671235 C317.44991,31.671235 317.689494,31.6504017 317.991577,31.608735 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="447.53366 10.47244 518.43127 10.47244 518.43127 41.842511 447.53366 41.842511"></polygon>
-                <path d="M471.62164,31.49936 L471.62164,33.077485 L462.793545,33.077485 C462.783129,32.6816517 462.845629,32.3014433 462.981045,31.93686 C463.210212,31.3326933 463.569587,30.7389433 464.05917,30.15561 C464.55917,29.5722767 465.27792,28.9004017 466.21542,28.139985 C467.663337,26.9524799 468.642504,26.0097691 469.15292,25.3118524 C469.663337,24.6139357 469.918545,23.9576857 469.918545,23.3431024 C469.918545,22.6868524 469.68417,22.1399774 469.21542,21.7024774 C468.757087,21.2545607 468.15292,21.0306024 467.40292,21.0306024 C466.611254,21.0306024 465.975837,21.2701857 465.49667,21.7493524 C465.02792,22.2181024 464.793545,22.8691441 464.793545,23.7024774 L463.106045,23.5306024 C463.220629,22.2701857 463.65292,21.3118524 464.40292,20.6556024 C465.163337,19.9993524 466.173754,19.6712274 467.43417,19.6712274 C468.71542,19.6712274 469.731045,20.0253941 470.481045,20.7337274 C471.231025,21.4420607 471.606015,22.3222691 471.606015,23.3743524 C471.606015,23.9056024 471.49664,24.4316441 471.27789,24.9524774 C471.05916,25.4733107 470.694587,26.0201857 470.18417,26.5931024 C469.68417,27.1556024 468.84042,27.9368549 467.65292,28.93686 C466.673754,29.7597767 466.043545,30.3222767 465.762295,30.62436 C465.481045,30.9160267 465.251879,31.2076933 465.074795,31.49936 L471.62164,31.49936 Z M473.387997,26.4837274 C473.387997,24.9108107 473.549456,23.6451857 473.872372,22.6868524 C474.195289,21.7181024 474.674456,20.9733107 475.309872,20.4524774 C475.955706,19.9316441 476.762997,19.6712274 477.731747,19.6712274 C478.450497,19.6712274 479.080706,19.8170607 479.622372,20.1087274 C480.164039,20.3899774 480.606747,20.8066441 480.950497,21.3587274 C481.304664,21.9003941 481.580706,22.5618524 481.778622,23.3431024 C481.986956,24.1139357 482.091122,25.1608107 482.091122,26.4837274 C482.091122,28.0566491 481.929664,29.327485 481.606747,30.296235 C481.283831,31.2545683 480.799456,31.99936 480.153622,32.53061 C479.518206,33.0514433 478.710914,33.31186 477.731747,33.31186 C476.450497,33.31186 475.440081,32.8483183 474.700497,31.921235 C473.825497,30.8066517 473.387997,28.9941491 473.387997,26.4837274 Z M475.059872,26.4837274 C475.059872,28.6816491 475.315081,30.1451933 475.825497,30.87436 C476.346331,31.59311 476.981747,31.952485 477.731747,31.952485 C478.492164,31.952485 479.127581,31.5879017 479.637997,30.858735 C480.148414,30.1295683 480.403622,28.6712325 480.403622,26.4837274 C480.403622,24.2858107 480.148414,22.8274774 479.637997,22.1087274 C479.127581,21.3899774 478.486956,21.0306024 477.716122,21.0306024 C476.966122,21.0306024 476.367164,21.3483107 475.919247,21.9837274 C475.346331,22.7962274 475.059872,24.2962274 475.059872,26.4837274 Z M483.763729,26.4837274 C483.763729,24.9108107 483.925188,23.6451857 484.248104,22.6868524 C484.571021,21.7181024 485.050188,20.9733107 485.685604,20.4524774 C486.331438,19.9316441 487.138729,19.6712274 488.107479,19.6712274 C488.826229,19.6712274 489.456438,19.8170607 489.998104,20.1087274 C490.539771,20.3899774 490.982479,20.8066441 491.326229,21.3587274 C491.680396,21.9003941 491.956438,22.5618524 492.154354,23.3431024 C492.362688,24.1139357 492.466854,25.1608107 492.466854,26.4837274 C492.466854,28.0566491 492.305396,29.327485 491.982479,30.296235 C491.659563,31.2545683 491.175188,31.99936 490.529354,32.53061 C489.893938,33.0514433 489.086646,33.31186 488.107479,33.31186 C486.826229,33.31186 485.815813,32.8483183 485.076229,31.921235 C484.201229,30.8066517 483.763729,28.9941491 483.763729,26.4837274 Z M485.435604,26.4837274 C485.435604,28.6816491 485.690813,30.1451933 486.201229,30.87436 C486.722063,31.59311 487.357479,31.952485 488.107479,31.952485 C488.867896,31.952485 489.503313,31.5879017 490.013729,30.858735 C490.524146,30.1295683 490.779354,28.6712325 490.779354,26.4837274 C490.779354,24.2858107 490.524146,22.8274774 490.013729,22.1087274 C489.503313,21.3899774 488.862688,21.0306024 488.091854,21.0306024 C487.341854,21.0306024 486.742896,21.3483107 486.294979,21.9837274 C485.722063,22.7962274 485.435604,24.2962274 485.435604,26.4837274 Z M494.139461,26.4837274 C494.139461,24.9108107 494.30092,23.6451857 494.623836,22.6868524 C494.946753,21.7181024 495.42592,20.9733107 496.061336,20.4524774 C496.70717,19.9316441 497.514461,19.6712274 498.483211,19.6712274 C499.201961,19.6712274 499.83217,19.8170607 500.373836,20.1087274 C500.915503,20.3899774 501.358211,20.8066441 501.701961,21.3587274 C502.056128,21.9003941 502.33217,22.5618524 502.530086,23.3431024 C502.73842,24.1139357 502.842586,25.1608107 502.842586,26.4837274 C502.842586,28.0566491 502.681128,29.327485 502.358211,30.296235 C502.035295,31.2545683 501.55092,31.99936 500.905086,32.53061 C500.26967,33.0514433 499.462378,33.31186 498.483211,33.31186 C497.201961,33.31186 496.191545,32.8483183 495.451961,31.921235 C494.576961,30.8066517 494.139461,28.9941491 494.139461,26.4837274 Z M495.811336,26.4837274 C495.811336,28.6816491 496.066545,30.1451933 496.576961,30.87436 C497.097795,31.59311 497.733211,31.952485 498.483211,31.952485 C499.243628,31.952485 499.879045,31.5879017 500.389461,30.858735 C500.899878,30.1295683 501.155086,28.6712325 501.155086,26.4837274 C501.155086,24.2858107 500.899878,22.8274774 500.389461,22.1087274 C499.879045,21.3899774 499.23842,21.0306024 498.467586,21.0306024 C497.717586,21.0306024 497.118628,21.3483107 496.670711,21.9837274 C496.097795,22.7962274 495.811336,24.2962274 495.811336,26.4837274 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polyline id="Path" fill-opacity="0" fill="#000000" points="208.80545 39.14516 208.80545 106.29134 24.80545 106.29134 24.80545 39.20815"></polyline>
-                <polyline id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round" points="208.80545 57.14516 208.80545 106.29134 24.80545 106.29134 24.80545 39.20815"></polyline>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="213.76065 57.14516 208.80545 43.530872 203.85025 57.14516"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="53.1076 90.47244 170.14696 90.47244 170.14696 121.842511 53.1076 121.842511"></polygon>
-                <path d="M67.43531,113.07748 L67.43531,99.718105 L72.482185,99.718105 C73.3676017,99.718105 74.044685,99.7597717 74.513435,99.843105 C75.1592683,99.9576883 75.7061433,100.17123 76.15406,100.48373 C76.6019767,100.785813 76.9613517,101.212897 77.232185,101.76498 C77.5030183,102.317063 77.638435,102.926438 77.638435,103.593105 C77.638435,104.728522 77.2738517,105.692063 76.544685,106.48373 C75.825935,107.26498 74.5186433,107.655605 72.62281,107.655605 L69.200935,107.655605 L69.200935,113.07748 L67.43531,113.07748 Z M69.200935,106.07748 L72.65406,106.07748 C73.7998933,106.07748 74.6123933,105.863938 75.09156,105.436855 C75.5707267,105.009772 75.81031,104.410813 75.81031,103.63998 C75.81031,103.07748 75.669685,102.598313 75.388435,102.20248 C75.107185,101.806647 74.732185,101.54623 74.263435,101.42123 C73.9717683,101.337897 73.4248933,101.29623 72.62281,101.29623 L69.200935,101.29623 L69.200935,106.07748 Z M79.066383,108.23373 C79.066383,106.442063 79.5611747,105.119147 80.550758,104.26498 C81.3840913,103.54623 82.3997163,103.186855 83.597633,103.186855 C84.9309663,103.186855 86.019508,103.624355 86.863258,104.499355 C87.7070029,105.363938 88.1288754,106.567063 88.1288754,108.10873 C88.1288754,109.348313 87.9413754,110.32748 87.566383,111.04623 C87.1913754,111.754563 86.6445029,112.306647 85.925758,112.70248 C85.2174247,113.098313 84.441383,113.29623 83.597633,113.29623 C82.2434663,113.29623 81.1497163,112.863938 80.316383,111.999355 C79.4830497,111.124355 79.066383,109.869147 79.066383,108.23373 Z M80.753883,108.23373 C80.753883,109.473313 81.019508,110.405605 81.550758,111.030605 C82.0924247,111.645188 82.7747163,111.95248 83.597633,111.95248 C84.410133,111.95248 85.0872163,111.645188 85.628883,111.030605 C86.1705497,110.405605 86.441383,109.457688 86.441383,108.186855 C86.441383,106.988938 86.1705497,106.082688 85.628883,105.468105 C85.0872163,104.853522 84.410133,104.54623 83.597633,104.54623 C82.7747163,104.54623 82.0924247,104.853522 81.550758,105.468105 C81.019508,106.072272 80.753883,106.994147 80.753883,108.23373 Z M91.832718,113.07748 L88.863968,103.405605 L90.567093,103.405605 L92.098343,108.98373 L92.676468,111.061855 C92.707718,110.957688 92.8743847,110.291022 93.176468,109.061855 L94.723343,103.405605 L96.410843,103.405605 L97.848343,109.01498 L98.332718,110.85873 L98.895218,108.999355 L100.551468,103.405605 L102.145218,103.405605 L99.113968,113.07748 L97.410843,113.07748 L95.879593,107.280605 L95.504593,105.63998 L93.551468,113.07748 L91.832718,113.07748 Z M110.149414,109.968105 L111.836914,110.17123 C111.566081,111.160813 111.071289,111.931647 110.352539,112.48373 C109.633789,113.025397 108.711914,113.29623 107.586914,113.29623 C106.170247,113.29623 105.045247,112.863938 104.211914,111.999355 C103.388997,111.124355 102.977539,109.900397 102.977539,108.32748 C102.977539,106.692063 103.394206,105.426438 104.227539,104.530605 C105.071289,103.634772 106.159831,103.186855 107.493164,103.186855 C108.784831,103.186855 109.836914,103.629563 110.649414,104.51498 C111.472331,105.38998 111.883789,106.624355 111.883789,108.218105 C111.883789,108.322272 111.883789,108.468105 111.883789,108.655605 L104.665039,108.655605 C104.727539,109.718105 105.029622,110.535813 105.571289,111.10873 C106.112956,111.67123 106.784831,111.95248 107.586914,111.95248 C108.191081,111.95248 108.706706,111.79623 109.133789,111.48373 C109.560872,111.160813 109.899414,110.655605 110.149414,109.968105 Z M104.758789,107.311855 L110.165039,107.311855 C110.092122,106.499355 109.883789,105.88998 109.540039,105.48373 C109.019206,104.848313 108.342122,104.530605 107.508789,104.530605 C106.758789,104.530605 106.123372,104.785813 105.602539,105.29623 C105.092122,105.79623 104.810872,106.468105 104.758789,107.311855 Z M113.884506,113.07748 L113.884506,103.405605 L115.353256,103.405605 L115.353256,104.874355 C115.728256,104.186855 116.072006,103.73373 116.384506,103.51498 C116.707423,103.29623 117.061589,103.186855 117.447006,103.186855 C117.999089,103.186855 118.561589,103.363938 119.134506,103.718105 L118.572006,105.23373 C118.165756,104.994147 117.764714,104.874355 117.368881,104.874355 C117.004298,104.874355 116.681381,104.98373 116.400131,105.20248 C116.118881,105.42123 115.915756,105.718105 115.790756,106.093105 C115.603256,106.676438 115.509506,107.317063 115.509506,108.01498 L115.509506,113.07748 L113.884506,113.07748 Z M119.722183,108.780605 L121.378433,108.63998 C121.461766,109.306647 121.644058,109.853522 121.925308,110.280605 C122.216975,110.707688 122.664891,111.056647 123.269058,111.32748 C123.883641,111.587897 124.571141,111.718105 125.331558,111.718105 C125.998225,111.718105 126.591975,111.619147 127.112808,111.42123 C127.633641,111.223313 128.019058,110.95248 128.269058,110.60873 C128.519058,110.254563 128.644058,109.869147 128.644058,109.45248 C128.644058,109.035813 128.519058,108.67123 128.269058,108.35873 C128.029475,108.04623 127.633641,107.780605 127.081558,107.561855 C126.716975,107.426438 125.9201,107.212897 124.690933,106.92123 C123.472183,106.619147 122.618016,106.337897 122.128433,106.07748 C121.4826,105.744147 121.003433,105.332688 120.690933,104.843105 C120.378433,104.343105 120.222183,103.785813 120.222183,103.17123 C120.222183,102.504563 120.414891,101.879563 120.800308,101.29623 C121.185725,100.70248 121.743016,100.254563 122.472183,99.95248 C123.211766,99.6503967 124.029475,99.499355 124.925308,99.499355 C125.914891,99.499355 126.784683,99.6608133 127.534683,99.98373 C128.2951,100.29623 128.878433,100.76498 129.284683,101.38998 C129.690933,102.004563 129.909683,102.70248 129.940933,103.48373 L128.253433,103.60873 C128.159683,102.76498 127.847183,102.129563 127.315933,101.70248 C126.7951,101.26498 126.024266,101.04623 125.003433,101.04623 C123.930516,101.04623 123.149266,101.244147 122.659683,101.63998 C122.1701,102.035813 121.925308,102.509772 121.925308,103.061855 C121.925308,103.541022 122.102391,103.931647 122.456558,104.23373 C122.789891,104.54623 123.675308,104.869147 125.112808,105.20248 C126.550308,105.525397 127.534683,105.806647 128.065933,106.04623 C128.847183,106.400397 129.4201,106.853522 129.784683,107.405605 C130.159683,107.957688 130.347183,108.593105 130.347183,109.311855 C130.347183,110.020188 130.144058,110.692063 129.737808,111.32748 C129.331558,111.95248 128.748225,112.442063 127.987808,112.79623 C127.227391,113.13998 126.368016,113.311855 125.409683,113.311855 C124.20135,113.311855 123.185725,113.134772 122.362808,112.780605 C121.550308,112.426438 120.909683,111.895188 120.440933,111.186855 C119.9826,110.478522 119.743016,109.676438 119.722183,108.780605 Z M132.556381,113.07748 L132.556381,99.718105 L134.197006,99.718105 L134.197006,104.51498 C134.957423,103.629563 135.920964,103.186855 137.087631,103.186855 C137.806381,103.186855 138.426173,103.32748 138.947006,103.60873 C139.478256,103.88998 139.858464,104.280605 140.087631,104.780605 C140.316798,105.280605 140.431381,106.004563 140.431381,106.95248 L140.431381,113.07748 L138.790756,113.07748 L138.790756,106.95248 C138.790756,106.129563 138.613673,105.530605 138.259506,105.155605 C137.905339,104.780605 137.400131,104.593105 136.743881,104.593105 C136.264714,104.593105 135.811589,104.723313 135.384506,104.98373 C134.957423,105.23373 134.650131,105.572272 134.462631,105.999355 C134.285548,106.426438 134.197006,107.020188 134.197006,107.780605 L134.197006,113.07748 L132.556381,113.07748 Z M142.322723,108.23373 C142.322723,106.442063 142.817515,105.119147 143.807098,104.26498 C144.640431,103.54623 145.656056,103.186855 146.853973,103.186855 C148.187306,103.186855 149.275848,103.624355 150.119598,104.499355 C150.963348,105.363938 151.385223,106.567063 151.385223,108.10873 C151.385223,109.348313 151.197723,110.32748 150.822723,111.04623 C150.447723,111.754563 149.900848,112.306647 149.182098,112.70248 C148.473765,113.098313 147.697723,113.29623 146.853973,113.29623 C145.499806,113.29623 144.406056,112.863938 143.572723,111.999355 C142.73939,111.124355 142.322723,109.869147 142.322723,108.23373 Z M144.010223,108.23373 C144.010223,109.473313 144.275848,110.405605 144.807098,111.030605 C145.348765,111.645188 146.031056,111.95248 146.853973,111.95248 C147.666473,111.95248 148.343556,111.645188 148.885223,111.030605 C149.42689,110.405605 149.697723,109.457688 149.697723,108.186855 C149.697723,106.988938 149.42689,106.082688 148.885223,105.468105 C148.343556,104.853522 147.666473,104.54623 146.853973,104.54623 C146.031056,104.54623 145.348765,104.853522 144.807098,105.468105 C144.275848,106.072272 144.010223,106.994147 144.010223,108.23373 Z M156.88594,111.60873 L157.120315,113.061855 C156.661982,113.155605 156.250523,113.20248 155.88594,113.20248 C155.29219,113.20248 154.828648,113.10873 154.495315,112.92123 C154.172398,112.73373 153.943232,112.488938 153.807815,112.186855 C153.672398,111.874355 153.60469,111.228522 153.60469,110.249355 L153.60469,104.67123 L152.401565,104.67123 L152.401565,103.405605 L153.60469,103.405605 L153.60469,101.01498 L155.22969,100.030605 L155.22969,103.405605 L156.88594,103.405605 L156.88594,104.67123 L155.22969,104.67123 L155.22969,110.343105 C155.22969,110.801438 155.255732,111.098313 155.307815,111.23373 C155.370315,111.369147 155.464065,111.478522 155.589065,111.561855 C155.724482,111.634772 155.91719,111.67123 156.16719,111.67123 C156.344273,111.67123 156.583857,111.650397 156.88594,111.60873 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-            </g>
-        </g>
-    </g>
-</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-wdg.svg b/docs/reference/images/analysis/token-graph-wdg.svg
deleted file mode 100644
index 992637bd668d5..0000000000000
--- a/docs/reference/images/analysis/token-graph-wdg.svg
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="602px" height="123px" viewBox="0 0 602 123" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
-    <title>Slice 1</title>
-    <desc>Created with Sketch.</desc>
-    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-        <g id="Analysis-(2)" transform="translate(1.000000, 1.000000)">
-            <g id="Group">
-                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
-                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
-                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
-                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
-                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
-                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
-                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
-                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
-                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="15.088913 39.14516 34.52198 39.14516 34.52198 60.971927 15.088913 60.971927"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="199.08891 39.14516 218.52197 39.14516 218.52197 60.971927 199.08891 60.971927"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="73.96063 10.47244 152.35432 10.47244 152.35432 41.842511 73.96063 41.842511"></polygon>
-                <path d="M84.39813,33.07748 L84.39813,19.718097 L89.445005,19.718097 C90.3304217,19.718097 91.007505,19.7597637 91.476255,19.843097 C92.1220883,19.9576803 92.6689633,20.171222 93.11688,20.483722 C93.5647967,20.7858053 93.9241717,21.2128887 94.195005,21.764972 C94.4658383,22.3170553 94.601255,22.9264303 94.601255,23.593097 C94.601255,24.7285137 94.2366717,25.6920553 93.507505,26.483722 C92.788755,27.264972 91.4814633,27.655597 89.58563,27.655597 L86.163755,27.655597 L86.163755,33.07748 L84.39813,33.07748 L84.39813,33.07748 Z M86.163755,26.0774724 L89.61688,26.0774724 C90.7627133,26.0774724 91.5752133,25.8639307 92.05438,25.4368474 C92.5335467,25.0097641 92.77313,24.4108057 92.77313,23.6399724 C92.77313,23.0774724 92.632505,22.5983057 92.351255,22.2024724 C92.070005,21.8066391 91.695005,21.5462224 91.226255,21.4212224 C90.9345883,21.3378891 90.3877133,21.2962224 89.58563,21.2962224 L86.163755,21.2962224 L86.163755,26.0774724 L86.163755,26.0774724 Z M96.029203,28.23373 C96.029203,26.4420583 96.5239947,25.1191391 97.513578,24.2649724 C98.3469113,23.5462224 99.3625363,23.1868474 100.560453,23.1868474 C101.893786,23.1868474 102.982328,23.6243474 103.826078,24.4993474 C104.669828,25.3639307 105.091703,26.5670583 105.091703,28.10873 C105.091703,29.3483133 104.904203,30.32748 104.529203,31.04623 C104.154203,31.7545633 103.607328,32.3066467 102.888578,32.70248 C102.180245,33.0983133 101.404203,33.29623 100.560453,33.29623 C99.2062863,33.29623 98.1125363,32.8639383 97.279203,31.999355 C96.4458697,31.124355 96.029203,29.8691467 96.029203,28.23373 Z M97.716703,28.23373 C97.716703,29.4733133 97.982328,30.405605 98.513578,31.030605 C99.0552447,31.6451883 99.7375363,31.95248 100.560453,31.95248 C101.372953,31.95248 102.050036,31.6451883 102.591703,31.030605 C103.13337,30.405605 103.404203,29.4576883 103.404203,28.186855 C103.404203,26.9889333 103.13337,26.0826807 102.591703,25.4680974 C102.050036,24.8535141 101.372953,24.5462224 100.560453,24.5462224 C99.7375363,24.5462224 99.0552447,24.8535141 98.513578,25.4680974 C97.982328,26.0722641 97.716703,26.9941416 97.716703,28.23373 Z M108.795545,33.07748 L105.826795,23.405597 L107.52992,23.405597 L109.06117,28.9837296 L109.639295,31.0618546 C109.670545,30.9576879 109.837212,30.2910213 110.139295,29.0618546 L111.68617,23.405597 L113.37367,23.405597 L114.81117,29.0149796 L115.295545,30.8587296 L115.858045,28.9993546 L117.514295,23.405597 L119.108045,23.405597 L116.076795,33.07748 L114.37367,33.07748 L112.84242,27.2805974 L112.46742,25.6399724 L110.514295,33.07748 L108.795545,33.07748 Z M127.112241,29.968105 L128.799741,30.17123 C128.528908,31.1608133 128.034116,31.9316467 127.315366,32.48373 C126.596616,33.0253967 125.674741,33.29623 124.549741,33.29623 C123.133074,33.29623 122.008074,32.8639383 121.174741,31.999355 C120.351824,31.124355 119.940366,29.9003967 119.940366,28.32748 C119.940366,26.6920583 120.357033,25.4264307 121.190366,24.5305974 C122.034116,23.6347641 123.122658,23.1868474 124.455991,23.1868474 C125.747658,23.1868474 126.799741,23.6295557 127.612241,24.5149724 C128.435158,25.3899724 128.846616,26.6243499 128.846616,28.218105 C128.846616,28.3222717 128.846616,28.468105 128.846616,28.655605 L121.627866,28.655605 C121.690366,29.718105 121.992449,30.5358133 122.534116,31.10873 C123.075783,31.67123 123.747658,31.95248 124.549741,31.95248 C125.153908,31.95248 125.669533,31.79623 126.096616,31.48373 C126.523699,31.1608133 126.862241,30.655605 127.112241,29.968105 Z M121.721616,27.3118474 L127.127866,27.3118474 C127.054949,26.4993474 126.846616,25.8899724 126.502866,25.4837224 C125.982033,24.8483057 125.304949,24.5305974 124.471616,24.5305974 C123.721616,24.5305974 123.086199,24.7858057 122.565366,25.2962224 C122.054949,25.7962224 121.773699,26.4680974 121.721616,27.3118474 Z M130.847333,33.07748 L130.847333,23.405597 L132.316083,23.405597 L132.316083,24.874347 C132.691083,24.186847 133.034833,23.733722 133.347333,23.514972 C133.67025,23.296222 134.024416,23.186847 134.409833,23.186847 C134.961916,23.186847 135.524416,23.3639303 136.097333,23.718097 L135.534833,25.233722 C135.128583,24.9941387 134.727541,24.874347 134.331708,24.874347 C133.967125,24.874347 133.644208,24.983722 133.362958,25.202472 C133.081708,25.421222 132.878583,25.718097 132.753583,26.093097 C132.566083,26.6764303 132.472333,27.3170579 132.472333,28.0149796 L132.472333,33.07748 L130.847333,33.07748 L130.847333,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="265.96063 10.47244 332.38583 10.47244 332.38583 41.842511 265.96063 41.842511"></polygon>
-                <path d="M280.82782,28.78061 L282.48407,28.639985 C282.567403,29.3066517 282.749695,29.8535267 283.030945,30.28061 C283.322612,30.7076933 283.770528,31.0566517 284.374695,31.327485 C284.989278,31.5879017 285.676778,31.71811 286.437195,31.71811 C287.103862,31.71811 287.697612,31.6191517 288.218445,31.421235 C288.739278,31.2233183 289.124695,30.952485 289.374695,30.608735 C289.624695,30.2545683 289.749695,29.8691517 289.749695,29.452485 C289.749695,29.0358183 289.624695,28.671235 289.374695,28.358735 C289.135112,28.0462299 288.739278,27.7806024 288.187195,27.5618524 C287.822612,27.4264357 287.025737,27.2128941 285.79657,26.9212274 C284.57782,26.6191441 283.723653,26.3378941 283.23407,26.0774774 C282.588237,25.7441441 282.10907,25.3326857 281.79657,24.8431024 C281.48407,24.3431024 281.32782,23.7858107 281.32782,23.1712274 C281.32782,22.5045607 281.520528,21.8795607 281.905945,21.2962274 C282.291362,20.7024774 282.848653,20.2545607 283.57782,19.9524774 C284.317403,19.6503941 285.135112,19.4993524 286.030945,19.4993524 C287.020528,19.4993524 287.89032,19.6608107 288.64032,19.9837274 C289.400737,20.2962274 289.98407,20.7649774 290.39032,21.3899774 C290.79657,22.0045607 291.01532,22.7024774 291.04657,23.4837274 L289.35907,23.6087274 C289.26532,22.7649774 288.95282,22.1295607 288.42157,21.7024774 C287.900737,21.2649774 287.129903,21.0462274 286.10907,21.0462274 C285.036153,21.0462274 284.254903,21.2441441 283.76532,21.6399774 C283.275737,22.0358107 283.030945,22.5097691 283.030945,23.0618524 C283.030945,23.5410191 283.208028,23.9316441 283.562195,24.2337274 C283.895528,24.5462274 284.780945,24.8691441 286.218445,25.2024774 C287.655945,25.5253941 288.64032,25.8066441 289.17157,26.0462274 C289.95282,26.4003941 290.525737,26.8535191 290.89032,27.4056024 C291.26532,27.9576908 291.45282,28.59311 291.45282,29.31186 C291.45282,30.0201933 291.249695,30.6920683 290.843445,31.327485 C290.437195,31.952485 289.853862,32.4420683 289.093445,32.796235 C288.333028,33.139985 287.473653,33.31186 286.51532,33.31186 C285.306987,33.31186 284.291362,33.1347767 283.468445,32.78061 C282.655945,32.4264433 282.01532,31.8951933 281.54657,31.18686 C281.088237,30.4785267 280.848653,29.6764433 280.82782,28.78061 Z M293.662018,33.077485 L293.662018,19.718102 L295.302643,19.718102 L295.302643,24.514977 C296.06306,23.6295603 297.026601,23.186852 298.193268,23.186852 C298.912018,23.186852 299.53181,23.327477 300.052643,23.608727 C300.583893,23.889977 300.964101,24.280602 301.193268,24.780602 C301.422435,25.280602 301.537018,26.0045603 301.537018,26.952477 L301.537018,33.077485 L299.896393,33.077485 L299.896393,26.952477 C299.896393,26.1295603 299.71931,25.530602 299.365143,25.155602 C299.010976,24.780602 298.505768,24.593102 297.849518,24.593102 C297.370351,24.593102 296.917226,24.7233103 296.490143,24.983727 C296.06306,25.233727 295.755768,25.5722687 295.568268,25.999352 C295.391185,26.4264353 295.302643,27.0201853 295.302643,27.780602 L295.302643,33.077485 L293.662018,33.077485 L293.662018,33.077485 Z M303.428375,28.233735 C303.428375,26.4420633 303.923167,25.1191441 304.91275,24.2649774 C305.746083,23.5462274 306.761708,23.1868524 307.959625,23.1868524 C309.292958,23.1868524 310.3815,23.6243524 311.22525,24.4993524 C312.069,25.3639357 312.490875,26.5670633 312.490875,28.108735 C312.490875,29.3483183 312.303375,30.327485 311.928375,31.046235 C311.553375,31.7545683 311.0065,32.3066517 310.28775,32.702485 C309.579417,33.0983183 308.803375,33.296235 307.959625,33.296235 C306.605458,33.296235 305.511708,32.8639433 304.678375,31.99936 C303.845042,31.12436 303.428375,29.8691517 303.428375,28.233735 Z M305.115875,28.233735 C305.115875,29.4733183 305.3815,30.40561 305.91275,31.03061 C306.454417,31.6451933 307.136708,31.952485 307.959625,31.952485 C308.772125,31.952485 309.449208,31.6451933 309.990875,31.03061 C310.532542,30.40561 310.803375,29.4576933 310.803375,28.18686 C310.803375,26.9889383 310.532542,26.0826857 309.990875,25.4681024 C309.449208,24.8535191 308.772125,24.5462274 307.959625,24.5462274 C307.136708,24.5462274 306.454417,24.8535191 305.91275,25.4681024 C305.3815,26.0722691 305.115875,26.9941466 305.115875,28.233735 Z M317.991577,31.608735 L318.225952,33.06186 C317.767619,33.15561 317.35616,33.202485 316.991577,33.202485 C316.397827,33.202485 315.934285,33.108735 315.600952,32.921235 C315.278035,32.733735 315.048869,32.4889433 314.913452,32.18686 C314.778035,31.87436 314.710327,31.2285267 314.710327,30.24936 L314.710327,24.6712274 L313.507202,24.6712274 L313.507202,23.4056024 L314.710327,23.4056024 L314.710327,21.0149774 L316.335327,20.0306024 L316.335327,23.4056024 L317.991577,23.4056024 L317.991577,24.6712274 L316.335327,24.6712274 L316.335327,30.34311 C316.335327,30.8014433 316.361369,31.0983183 316.413452,31.233735 C316.475952,31.3691517 316.569702,31.4785267 316.694702,31.56186 C316.830119,31.6347767 317.022827,31.671235 317.272827,31.671235 C317.44991,31.671235 317.689494,31.6504017 317.991577,31.608735 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="447.53366 10.47244 518.43127 10.47244 518.43127 41.842511 447.53366 41.842511"></polygon>
-                <path d="M471.62164,31.49936 L471.62164,33.077485 L462.793545,33.077485 C462.783129,32.6816517 462.845629,32.3014433 462.981045,31.93686 C463.210212,31.3326933 463.569587,30.7389433 464.05917,30.15561 C464.55917,29.5722767 465.27792,28.9004017 466.21542,28.139985 C467.663337,26.9524799 468.642504,26.0097691 469.15292,25.3118524 C469.663337,24.6139357 469.918545,23.9576857 469.918545,23.3431024 C469.918545,22.6868524 469.68417,22.1399774 469.21542,21.7024774 C468.757087,21.2545607 468.15292,21.0306024 467.40292,21.0306024 C466.611254,21.0306024 465.975837,21.2701857 465.49667,21.7493524 C465.02792,22.2181024 464.793545,22.8691441 464.793545,23.7024774 L463.106045,23.5306024 C463.220629,22.2701857 463.65292,21.3118524 464.40292,20.6556024 C465.163337,19.9993524 466.173754,19.6712274 467.43417,19.6712274 C468.71542,19.6712274 469.731045,20.0253941 470.481045,20.7337274 C471.231025,21.4420607 471.606015,22.3222691 471.606015,23.3743524 C471.606015,23.9056024 471.49664,24.4316441 471.27789,24.9524774 C471.05916,25.4733107 470.694587,26.0201857 470.18417,26.5931024 C469.68417,27.1556024 468.84042,27.9368549 467.65292,28.93686 C466.673754,29.7597767 466.043545,30.3222767 465.762295,30.62436 C465.481045,30.9160267 465.251879,31.2076933 465.074795,31.49936 L471.62164,31.49936 Z M473.387997,26.4837274 C473.387997,24.9108107 473.549456,23.6451857 473.872372,22.6868524 C474.195289,21.7181024 474.674456,20.9733107 475.309872,20.4524774 C475.955706,19.9316441 476.762997,19.6712274 477.731747,19.6712274 C478.450497,19.6712274 479.080706,19.8170607 479.622372,20.1087274 C480.164039,20.3899774 480.606747,20.8066441 480.950497,21.3587274 C481.304664,21.9003941 481.580706,22.5618524 481.778622,23.3431024 C481.986956,24.1139357 482.091122,25.1608107 482.091122,26.4837274 C482.091122,28.0566491 481.929664,29.327485 481.606747,30.296235 C481.283831,31.2545683 480.799456,31.99936 480.153622,32.53061 C479.518206,33.0514433 478.710914,33.31186 477.731747,33.31186 C476.450497,33.31186 475.440081,32.8483183 474.700497,31.921235 C473.825497,30.8066517 473.387997,28.9941491 473.387997,26.4837274 Z M475.059872,26.4837274 C475.059872,28.6816491 475.315081,30.1451933 475.825497,30.87436 C476.346331,31.59311 476.981747,31.952485 477.731747,31.952485 C478.492164,31.952485 479.127581,31.5879017 479.637997,30.858735 C480.148414,30.1295683 480.403622,28.6712325 480.403622,26.4837274 C480.403622,24.2858107 480.148414,22.8274774 479.637997,22.1087274 C479.127581,21.3899774 478.486956,21.0306024 477.716122,21.0306024 C476.966122,21.0306024 476.367164,21.3483107 475.919247,21.9837274 C475.346331,22.7962274 475.059872,24.2962274 475.059872,26.4837274 Z M483.763729,26.4837274 C483.763729,24.9108107 483.925188,23.6451857 484.248104,22.6868524 C484.571021,21.7181024 485.050188,20.9733107 485.685604,20.4524774 C486.331438,19.9316441 487.138729,19.6712274 488.107479,19.6712274 C488.826229,19.6712274 489.456438,19.8170607 489.998104,20.1087274 C490.539771,20.3899774 490.982479,20.8066441 491.326229,21.3587274 C491.680396,21.9003941 491.956438,22.5618524 492.154354,23.3431024 C492.362688,24.1139357 492.466854,25.1608107 492.466854,26.4837274 C492.466854,28.0566491 492.305396,29.327485 491.982479,30.296235 C491.659563,31.2545683 491.175188,31.99936 490.529354,32.53061 C489.893938,33.0514433 489.086646,33.31186 488.107479,33.31186 C486.826229,33.31186 485.815813,32.8483183 485.076229,31.921235 C484.201229,30.8066517 483.763729,28.9941491 483.763729,26.4837274 Z M485.435604,26.4837274 C485.435604,28.6816491 485.690813,30.1451933 486.201229,30.87436 C486.722063,31.59311 487.357479,31.952485 488.107479,31.952485 C488.867896,31.952485 489.503313,31.5879017 490.013729,30.858735 C490.524146,30.1295683 490.779354,28.6712325 490.779354,26.4837274 C490.779354,24.2858107 490.524146,22.8274774 490.013729,22.1087274 C489.503313,21.3899774 488.862688,21.0306024 488.091854,21.0306024 C487.341854,21.0306024 486.742896,21.3483107 486.294979,21.9837274 C485.722063,22.7962274 485.435604,24.2962274 485.435604,26.4837274 Z M494.139461,26.4837274 C494.139461,24.9108107 494.30092,23.6451857 494.623836,22.6868524 C494.946753,21.7181024 495.42592,20.9733107 496.061336,20.4524774 C496.70717,19.9316441 497.514461,19.6712274 498.483211,19.6712274 C499.201961,19.6712274 499.83217,19.8170607 500.373836,20.1087274 C500.915503,20.3899774 501.358211,20.8066441 501.701961,21.3587274 C502.056128,21.9003941 502.33217,22.5618524 502.530086,23.3431024 C502.73842,24.1139357 502.842586,25.1608107 502.842586,26.4837274 C502.842586,28.0566491 502.681128,29.327485 502.358211,30.296235 C502.035295,31.2545683 501.55092,31.99936 500.905086,32.53061 C500.26967,33.0514433 499.462378,33.31186 498.483211,33.31186 C497.201961,33.31186 496.191545,32.8483183 495.451961,31.921235 C494.576961,30.8066517 494.139461,28.9941491 494.139461,26.4837274 Z M495.811336,26.4837274 C495.811336,28.6816491 496.066545,30.1451933 496.576961,30.87436 C497.097795,31.59311 497.733211,31.952485 498.483211,31.952485 C499.243628,31.952485 499.879045,31.5879017 500.389461,30.858735 C500.899878,30.1295683 501.155086,28.6712325 501.155086,26.4837274 C501.155086,24.2858107 500.899878,22.8274774 500.389461,22.1087274 C499.879045,21.3899774 499.23842,21.0306024 498.467586,21.0306024 C497.717586,21.0306024 497.118628,21.3483107 496.670711,21.9837274 C496.097795,22.7962274 495.811336,24.2962274 495.811336,26.4837274 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-                <polygon id="Path" fill="#FFFFFF" points="380.66196 40.35864 400.095035 40.35864 400.095035 62.185407 380.66196 62.185407"></polygon>
-                <polyline id="Path" fill-opacity="0" fill="#000000" points="390.37848 40.35864 390.37848 106.29133 24.80368 106.29133 24.80368 39.13029"></polyline>
-                <polyline id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round" points="390.37848 58.35864 390.37848 106.291333 24.80368 106.291333 24.80368 39.130293"></polyline>
-                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="395.33368 58.35864 390.37848 44.744336 385.42328 58.35864"></polygon>
-                <polygon id="Path" fill="#FFFFFF" points="149.10759 90.47244 266.14697 90.47244 266.14697 121.842511 149.10759 121.842511"></polygon>
-                <path d="M163.4353,113.07748 L163.4353,99.718105 L168.482175,99.718105 C169.367592,99.718105 170.044675,99.7597717 170.513425,99.843105 C171.159258,99.9576883 171.706133,100.17123 172.15405,100.48373 C172.601967,100.785813 172.961342,101.212897 173.232175,101.76498 C173.503008,102.317063 173.638425,102.926438 173.638425,103.593105 C173.638425,104.728522 173.273842,105.692063 172.544675,106.48373 C171.825925,107.26498 170.518633,107.655605 168.6228,107.655605 L165.200925,107.655605 L165.200925,113.07748 L163.4353,113.07748 Z M165.200925,106.07748 L168.65405,106.07748 C169.799883,106.07748 170.612383,105.863938 171.09155,105.436855 C171.570717,105.009772 171.8103,104.410813 171.8103,103.63998 C171.8103,103.07748 171.669675,102.598313 171.388425,102.20248 C171.107175,101.806647 170.732175,101.54623 170.263425,101.42123 C169.971758,101.337897 169.424883,101.29623 168.6228,101.29623 L165.200925,101.29623 L165.200925,106.07748 Z M175.066373,108.23373 C175.066373,106.442063 175.561165,105.119147 176.550748,104.26498 C177.384081,103.54623 178.399706,103.186855 179.597623,103.186855 C180.930956,103.186855 182.019498,103.624355 182.863248,104.499355 C183.706998,105.363938 184.128873,106.567063 184.128873,108.10873 C184.128873,109.348313 183.941373,110.32748 183.566373,111.04623 C183.191373,111.754563 182.644498,112.306647 181.925748,112.70248 C181.217415,113.098313 180.441373,113.29623 179.597623,113.29623 C178.243456,113.29623 177.149706,112.863938 176.316373,111.999355 C175.48304,111.124355 175.066373,109.869147 175.066373,108.23373 Z M176.753873,108.23373 C176.753873,109.473313 177.019498,110.405605 177.550748,111.030605 C178.092415,111.645188 178.774706,111.95248 179.597623,111.95248 C180.410123,111.95248 181.087206,111.645188 181.628873,111.030605 C182.17054,110.405605 182.441373,109.457688 182.441373,108.186855 C182.441373,106.988938 182.17054,106.082688 181.628873,105.468105 C181.087206,104.853522 180.410123,104.54623 179.597623,104.54623 C178.774706,104.54623 178.092415,104.853522 177.550748,105.468105 C177.019498,106.072272 176.753873,106.994147 176.753873,108.23373 Z M187.832715,113.07748 L184.863965,103.405605 L186.56709,103.405605 L188.09834,108.98373 L188.676465,111.061855 C188.707715,110.957688 188.874382,110.291022 189.176465,109.061855 L190.72334,103.405605 L192.41084,103.405605 L193.84834,109.01498 L194.332715,110.85873 L194.895215,108.999355 L196.551465,103.405605 L198.145215,103.405605 L195.113965,113.07748 L193.41084,113.07748 L191.87959,107.280605 L191.50459,105.63998 L189.551465,113.07748 L187.832715,113.07748 Z M206.149411,109.968105 L207.836911,110.17123 C207.566078,111.160813 207.071286,111.931647 206.352536,112.48373 C205.633786,113.025397 204.711911,113.29623 203.586911,113.29623 C202.170244,113.29623 201.045244,112.863938 200.211911,111.999355 C199.388994,111.124355 198.977536,109.900397 198.977536,108.32748 C198.977536,106.692063 199.394203,105.426438 200.227536,104.530605 C201.071286,103.634772 202.159828,103.186855 203.493161,103.186855 C204.784828,103.186855 205.836911,103.629563 206.649411,104.51498 C207.472328,105.38998 207.883786,106.624355 207.883786,108.218105 C207.883786,108.322272 207.883786,108.468105 207.883786,108.655605 L200.665036,108.655605 C200.727536,109.718105 201.029619,110.535813 201.571286,111.10873 C202.112953,111.67123 202.784828,111.95248 203.586911,111.95248 C204.191078,111.95248 204.706703,111.79623 205.133786,111.48373 C205.560869,111.160813 205.899411,110.655605 206.149411,109.968105 Z M200.758786,107.311855 L206.165036,107.311855 C206.092119,106.499355 205.883786,105.88998 205.540036,105.48373 C205.019203,104.848313 204.342119,104.530605 203.508786,104.530605 C202.758786,104.530605 202.123369,104.785813 201.602536,105.29623 C201.092119,105.79623 200.810869,106.468105 200.758786,107.311855 Z M209.884503,113.07748 L209.884503,103.405605 L211.353253,103.405605 L211.353253,104.874355 C211.728253,104.186855 212.072003,103.73373 212.384503,103.51498 C212.70742,103.29623 213.061586,103.186855 213.447003,103.186855 C213.999086,103.186855 214.561591,103.363938 215.134518,103.718105 L214.572003,105.23373 C214.165753,104.994147 213.764711,104.874355 213.368878,104.874355 C213.004295,104.874355 212.681378,104.98373 212.400128,105.20248 C212.118878,105.42123 211.915753,105.718105 211.790753,106.093105 C211.603253,106.676438 211.509503,107.317063 211.509503,108.01498 L211.509503,113.07748 L209.884503,113.07748 L209.884503,113.07748 Z M215.722195,108.780605 L217.378445,108.63998 C217.461779,109.306647 217.64407,109.853522 217.92532,110.280605 C218.216987,110.707688 218.664904,111.056647 219.26907,111.32748 C219.883654,111.587897 220.571154,111.718105 221.33157,111.718105 C221.998237,111.718105 222.591987,111.619147 223.11282,111.42123 C223.633654,111.223313 224.01907,110.95248 224.26907,110.60873 C224.51907,110.254563 224.64407,109.869147 224.64407,109.45248 C224.64407,109.035813 224.51907,108.67123 224.26907,108.35873 C224.029487,108.04623 223.633654,107.780605 223.08157,107.561855 C222.716987,107.426438 221.920112,107.212897 220.690945,106.92123 C219.472195,106.619147 218.618029,106.337897 218.128445,106.07748 C217.482612,105.744147 217.003445,105.332688 216.690945,104.843105 C216.378445,104.343105 216.222195,103.785813 216.222195,103.17123 C216.222195,102.504563 216.414904,101.879563 216.80032,101.29623 C217.185737,100.70248 217.743029,100.254563 218.472195,99.95248 C219.211779,99.6503967 220.029487,99.499355 220.92532,99.499355 C221.914904,99.499355 222.784695,99.6608133 223.534695,99.98373 C224.295112,100.29623 224.878445,100.76498 225.284695,101.38998 C225.690945,102.004563 225.909695,102.70248 225.940945,103.48373 L224.253445,103.60873 C224.159695,102.76498 223.847195,102.129563 223.315945,101.70248 C222.795112,101.26498 222.024279,101.04623 221.003445,101.04623 C219.930529,101.04623 219.149279,101.244147 218.659695,101.63998 C218.170112,102.035813 217.92532,102.509772 217.92532,103.061855 C217.92532,103.541022 218.102404,103.931647 218.45657,104.23373 C218.789904,104.54623 219.67532,104.869147 221.11282,105.20248 C222.55032,105.525397 223.534695,105.806647 224.065945,106.04623 C224.847195,106.400397 225.420112,106.853522 225.784695,107.405605 C226.159695,107.957688 226.347195,108.593105 226.347195,109.311855 C226.347195,110.020188 226.14407,110.692063 225.73782,111.32748 C225.33157,111.95248 224.748237,112.442063 223.98782,112.79623 C223.227404,113.13998 222.368029,113.311855 221.409695,113.311855 C220.201362,113.311855 219.185737,113.134772 218.36282,112.780605 C217.55032,112.426438 216.909695,111.895188 216.440945,111.186855 C215.982612,110.478522 215.743029,109.676438 215.722195,108.780605 Z M228.556393,113.07748 L228.556393,99.718105 L230.197018,99.718105 L230.197018,104.51498 C230.957435,103.629563 231.920977,103.186855 233.087643,103.186855 C233.806393,103.186855 234.426185,103.32748 234.947018,103.60873 C235.478268,103.88998 235.858477,104.280605 236.087643,104.780605 C236.31681,105.280605 236.431393,106.004563 236.431393,106.95248 L236.431393,113.07748 L234.790768,113.07748 L234.790768,106.95248 C234.790768,106.129563 234.613685,105.530605 234.259518,105.155605 C233.905352,104.780605 233.400143,104.593105 232.743893,104.593105 C232.264727,104.593105 231.811602,104.723313 231.384518,104.98373 C230.957435,105.23373 230.650143,105.572272 230.462643,105.999355 C230.28556,106.426438 230.197018,107.020188 230.197018,107.780605 L230.197018,113.07748 L228.556393,113.07748 Z M238.32272,108.23373 C238.32272,106.442063 238.817512,105.119147 239.807095,104.26498 C240.640429,103.54623 241.656054,103.186855 242.85397,103.186855 C244.187304,103.186855 245.275845,103.624355 246.119595,104.499355 C246.963345,105.363938 247.38522,106.567063 247.38522,108.10873 C247.38522,109.348313 247.19772,110.32748 246.82272,111.04623 C246.44772,111.754563 245.900845,112.306647 245.182095,112.70248 C244.473762,113.098313 243.69772,113.29623 242.85397,113.29623 C241.499804,113.29623 240.406054,112.863938 239.57272,111.999355 C238.739387,111.124355 238.32272,109.869147 238.32272,108.23373 Z M240.01022,108.23373 C240.01022,109.473313 240.275845,110.405605 240.807095,111.030605 C241.348762,111.645188 242.031054,111.95248 242.85397,111.95248 C243.66647,111.95248 244.343554,111.645188 244.88522,111.030605 C245.426887,110.405605 245.69772,109.457688 245.69772,108.186855 C245.69772,106.988938 245.426887,106.082688 244.88522,105.468105 C244.343554,104.853522 243.66647,104.54623 242.85397,104.54623 C242.031054,104.54623 241.348762,104.853522 240.807095,105.468105 C240.275845,106.072272 240.01022,106.994147 240.01022,108.23373 Z M252.885952,111.60873 L253.120327,113.061855 C252.661994,113.155605 252.250536,113.20248 251.885952,113.20248 C251.292202,113.20248 250.828661,113.10873 250.495327,112.92123 C250.172411,112.73373 249.943244,112.488938 249.807827,112.186855 C249.672411,111.874355 249.604702,111.228522 249.604702,110.249355 L249.604702,104.67123 L248.401577,104.67123 L248.401577,103.405605 L249.604702,103.405605 L249.604702,101.01498 L251.229702,100.030605 L251.229702,103.405605 L252.885952,103.405605 L252.885952,104.67123 L251.229702,104.67123 L251.229702,110.343105 C251.229702,110.801438 251.255744,111.098313 251.307827,111.23373 C251.370327,111.369147 251.464077,111.478522 251.589077,111.561855 C251.724494,111.634772 251.917202,111.67123 252.167202,111.67123 C252.344286,111.67123 252.583869,111.650397 252.885952,111.60873 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
-            </g>
-        </g>
-    </g>
-</svg>
\ No newline at end of file
diff --git a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
index a27ebecd9a9c7..7933f5805745e 100644
--- a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
+++ b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
@@ -67,11 +67,22 @@ If you use {es} {security-features}, you must have:
 Use the put enrich policy API
 to create a new <<enrich-policy,enrich policy>>.
 
+// tag::update-enrich-policy[]
 [WARNING]
 ====
-include::../../enrich.asciidoc[tag=update-enrich-policy]
-====
+Once created, you can't update or change an enrich policy.
+Instead, you can:
+
+.   Create and <<execute-enrich-policy-api,execute>> a new enrich policy.
 
+.   Replace the previous enrich policy
+    with the new enrich policy
+    in any in-use enrich processors.
+
+.   Use the <<delete-enrich-policy-api, delete enrich policy>> API
+    to delete the previous enrich policy.
+====
+// end::update-enrich-policy[]
 
 
 [[put-enrich-policy-api-path-params]]
diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc
index 98530032a9316..5a8e0dab3a25a 100644
--- a/docs/reference/ingest/enrich.asciidoc
+++ b/docs/reference/ingest/enrich.asciidoc
@@ -150,11 +150,7 @@ include::enrich.asciidoc[tag=enrich-policy-fields]
 You can use this definition to create the enrich policy with the
 <<put-enrich-policy-api,put enrich policy API>>.
 
-[WARNING]
-====
-Once created, you can't update or change an enrich policy.
-See <<update-enrich-policies>>.
-====
+include::apis/enrich/put-enrich-policy.asciidoc[tag=update-enrich-policy]
 
 [[execute-enrich-policy]]
 ==== Execute the enrich policy
@@ -218,19 +214,7 @@ using your ingest pipeline.
 [[update-enrich-policies]]
 ==== Update an enrich policy
 
-// tag::update-enrich-policy[]
-Once created, you can't update or change an enrich policy.
-Instead, you can:
-
-.   Create and <<execute-enrich-policy-api,execute>> a new enrich policy.
-
-.   Replace the previous enrich policy
-    with the new enrich policy
-    in any in-use enrich processors.
-
-.   Use the <<delete-enrich-policy-api, delete enrich policy>> API
-    to delete the previous enrich policy.
-// end::update-enrich-policy[]
+include::apis/enrich/put-enrich-policy.asciidoc[tag=update-enrich-policy]
 
 [role="xpack"]
 [testenv="basic"]
diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc
index b1b110e2b3889..ec3df1fa96347 100644
--- a/docs/reference/ingest/processors/inference.asciidoc
+++ b/docs/reference/ingest/processors/inference.asciidoc
@@ -14,7 +14,7 @@ ingested in the pipeline.
 | Name               | Required  | Default                        | Description
 | `model_id`         | yes       | -                              | (String) The ID of the model to load and infer against.
 | `target_field`     | no        | `ml.inference.<processor_tag>` | (String) Field added to incoming documents to contain results objects.
-| `field_map`        | yes       | -                              | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
+| `field_mappings`   | yes       | -                              | (Object) Maps the document field names to the known field names of the model.
 | `inference_config` | yes       | -                              | (Object) Contains the inference type and its options. There are two types: <<inference-processor-regression-opt,`regression`>> and <<inference-processor-classification-opt,`classification`>>.
 include::common-options.asciidoc[]
 |======
@@ -26,7 +26,7 @@ include::common-options.asciidoc[]
   "inference": {
     "model_id": "flight_delay_regression-1571767128603",
     "target_field": "FlightDelayMin_prediction_infer",
-    "field_map": {},
+    "field_mappings": {},
     "inference_config": { "regression": {} }
   }
 }
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index d26d897474deb..1b5afad31f0c6 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -7,7 +7,7 @@ document:
 [float]
 === Core datatypes
 
-string::         <<text,`text`>>, <<keyword,`keyword`>> and <<wildcard,`wildcard`>>
+string::         <<text,`text`>> and <<keyword,`keyword`>>
 <<number>>::     `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`
 <<date>>::       `date`
 <<date_nanos>>:: `date_nanos`
@@ -131,5 +131,3 @@ include::types/token-count.asciidoc[]
 include::types/shape.asciidoc[]
 
 include::types/constant-keyword.asciidoc[]
-
-include::types/wildcard.asciidoc[]
diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc
index 6d1dbe49b131d..440530b110247 100644
--- a/docs/reference/mapping/types/histogram.asciidoc
+++ b/docs/reference/mapping/types/histogram.asciidoc
@@ -69,6 +69,7 @@ The following <<indices-create-index, create index>> API request creates a new i
 * `my_histogram`, a `histogram` field used to store percentile data
 * `my_text`, a `keyword` field used to store a title for the histogram
 
+[ INSERT CREATE INDEX SNIPPET ]
 [source,console]
 --------------------------------------------------
 PUT my_index
diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc
index da4cb14ba9904..75cb76a4faff4 100644
--- a/docs/reference/mapping/types/search-as-you-type.asciidoc
+++ b/docs/reference/mapping/types/search-as-you-type.asciidoc
@@ -162,18 +162,15 @@ GET my_index/_search
 The following parameters are accepted in a mapping for the `search_as_you_type`
 field and are specific to this field type
 
+[horizontal]
+
 `max_shingle_size`::
-+
---
-(Optional, integer)
-Largest shingle size to create. Valid values are `2` (inclusive) to `4`
-(inclusive). Defaults to `3`.
 
-A subfield is created for each integer between `2` and this value. For example,
-a value of `3` creates two subfields: `my_field._2gram` and `my_field._3gram`
+    The largest shingle size to index the input with and create subfields for,
+    creating one subfield for each shingle size between 2 and
+    `max_shingle_size`. Accepts integer values between 2 and 4 inclusive. This
+    option defaults to 3.
 
-More subfields enables more specific queries but increases index size.
---
 
 [[general-params]]
 ==== Parameters of the field type as a text field
diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
deleted file mode 100644
index 51d10ff53ca92..0000000000000
--- a/docs/reference/mapping/types/wildcard.asciidoc
+++ /dev/null
@@ -1,53 +0,0 @@
-[role="xpack"]
-[testenv="basic"]
-[[wildcard]]
-=== Wildcard datatype
-++++
-<titleabbrev>Wildcard</titleabbrev>
-++++
-
-A `wildcard` field stores values optimised for wildcard grep-like queries.
-Wildcard queries are possible on other field types but suffer from constraints:
-* `text` fields limit matching of any wildcard expressions to individual tokens rather than the original whole value held in a field
-* `keyword` fields are untokenized but slow at performing wildcard queries (especially patterns with leading wildcards).
-
-Internally the `wildcard` field indexes the whole field value using ngrams and stores the full string.
-The index is used as a rough filter to cut down the number of values that are then checked by retrieving and checking the full values.
-This field is especially well suited to run grep-like queries on log lines. Storage costs are typically lower than those of `keyword`
-fields but search speeds for exact matches on full terms are slower.
-
-You index and search a wildcard field as follows
-
-[source,console]
---------------------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "properties": {
-      "my_wildcard": {
-        "type": "wildcard"
-      }
-    }
-  }
-}
-
-PUT my_index/_doc/1
-{
-  "my_wildcard" : "This string can be quite lengthy"
-}
-
-POST my_index/_doc/_search
-{
-  "query": {
-      "wildcard" : "*quite*lengthy"
-  }
-}
-
-
---------------------------------------------------
-
-
-==== Limitations
-
-* `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries.
-
diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc
index 9e52ddb770563..6a5b319616096 100644
--- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc
+++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc
@@ -94,10 +94,3 @@ breaking change was made necessary by
 https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[AWS's
 announcement] that the path-style access pattern is deprecated and will be
 unsupported on buckets created after September 30th 2020.
-
-[float]
-==== Restore requests no longer accept settings
-
-In earlier versions, you could pass both `settings` and `index_settings` in the
-body of a restore snapshot request, but the `settings` value was ignored. The
-restore snapshot API now rejects requests that include a `settings` value.
diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
index 6a011dcb04f48..d9087a6dac51b 100644
--- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
@@ -124,9 +124,9 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=eta]
 (Optional, double) 
 include::{docdir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction]
 
-`analysis`.`classification`.`max_trees`::::
+`analysis`.`classification`.`maximum_number_trees`::::
 (Optional, integer) 
-include::{docdir}/ml/ml-shared.asciidoc[tag=max-trees]
+include::{docdir}/ml/ml-shared.asciidoc[tag=maximum-number-trees]
 
 `analysis`.`classification`.`gamma`::::
 (Optional, double) 
@@ -136,10 +136,6 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=gamma]
 (Optional, double) 
 include::{docdir}/ml/ml-shared.asciidoc[tag=lambda]
 
-`analysis`.`classification`.`class_assignment_objective`::::
-(Optional, string)
-include::{docdir}/ml/ml-shared.asciidoc[tag=class-assignment-objective]
-
 `analysis`.`classification`.`num_top_classes`::::
 (Optional, integer)
 include::{docdir}/ml/ml-shared.asciidoc[tag=num-top-classes]
@@ -222,9 +218,9 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=eta]
 (Optional, double)
 include::{docdir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction]
 
-`analysis`.`regression`.`max_trees`::::
+`analysis`.`regression`.`maximum_number_trees`::::
 (Optional, integer) 
-include::{docdir}/ml/ml-shared.asciidoc[tag=max-trees]
+include::{docdir}/ml/ml-shared.asciidoc[tag=maximum-number-trees]
 
 `analysis`.`regression`.`gamma`::::
 (Optional, double) 
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index 16c9f486d8d63..ce0a06174a24a 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -339,14 +339,6 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=mode]
 include::{docdir}/ml/ml-shared.asciidoc[tag=time-span]
 end::chunking-config[]
 
-tag::class-assignment-objective[]
-Defines the objective to optimize when assigning class labels. Available
-objectives are `maximize_accuracy` and `maximize_minimum_recall`. When maximizing
-accuracy class labels are chosen to maximize the number of correct predictions.
-When maximizing minimum recall labels are chosen to maximize the minimum recall
-for any class. Defaults to maximize_minimum_recall.
-end::class-assignment-objective[]
-
 tag::custom-rules[]
 An array of custom rule objects, which enable you to customize the way detectors
 operate. For example, a rule may dictate to the detector conditions under which
@@ -954,10 +946,10 @@ remain started until it is explicitly stopped. By default this setting is not
 set.
 end::max-empty-searches[]
 
-tag::max-trees[]
+tag::maximum-number-trees[]
 Advanced configuration option. Defines the maximum number of trees the forest is
 allowed to contain. The maximum value is 2000.
-end::max-trees[]
+end::maximum-number-trees[]
 
 tag::memory-estimation[]
 An object containing the memory estimates. The object has the 
@@ -1325,11 +1317,6 @@ The configuration of how to source the analysis data. It requires an
   (Required, string or array) Index or indices on which to perform the 
   analysis. It can be a single index or index pattern as well as an array of 
   indices or patterns.
-+
---
-WARNING: If your source indices contain documents with the same IDs, only the 
-document that is indexed last appears in the destination index.
---
   
 `query`:::
   (Optional, object) The {es} query domain-specific language 
@@ -1519,17 +1506,6 @@ The estimated number of operations to use the trained model.
 `license_level`:::
 (string)
 The license level of the trained model.
-
-`default_field_map` :::
-(object)
-A string to string object that contains the default field map to use
-when inferring against the model. For example, data frame analytics
-may train the model on a specific multi-field `foo.keyword`.
-The analytics job would then supply a default field map entry for
-`"foo" : "foo.keyword"`.
-
-Any field map described in the inference configuration takes precedence.
-
 end::trained-model-configs[]
 
 tag::training-percent[]
diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc
index 69f78f1364c52..417acff5a5254 100644
--- a/docs/reference/modules/cross-cluster-search.asciidoc
+++ b/docs/reference/modules/cross-cluster-search.asciidoc
@@ -253,33 +253,30 @@ PUT _cluster/settings
 If `cluster_two` is disconnected or unavailable during a {ccs}, {es} won't
 include matching documents from that cluster in the final results.
 
+[discrete]
+[[ccs-works]]
+== How {ccs} works
+
+include::./remote-clusters.asciidoc[tag=how-remote-clusters-work]
+
 [discrete]
 [[ccs-gateway-seed-nodes]]
-== Selecting gateway and seed nodes in sniff mode
+=== Selecting gateway and seed nodes
 
-For remote clusters using the <<sniff-mode,sniff connection>> mode, gateway and
-seed nodes need to be accessible from the local cluster via your network.
+Gateway and seed nodes need to be accessible from the local cluster via your
+network.
 
-By default, any non-<<master-node,master-eligible>> node can act as a
-gateway node. If wanted, you can define the gateway nodes for a cluster by
-setting `cluster.remote.node.attr.gateway` to `true`.
+By default, any master-ineligible node can act as a gateway node. If wanted,
+you can define the gateway nodes for a cluster by setting
+`cluster.remote.node.attr.gateway` to `true`.
 
 For {ccs}, we recommend you use gateway nodes that are capable of serving as
 <<coordinating-node,coordinating nodes>> for search requests. If
 wanted, the seed nodes for a cluster can be a subset of these gateway nodes.
 
-[discrete]
-[[ccs-proxy-mode]]
-== {ccs} in proxy mode
-
-<<proxy-mode,Proxy mode>> remote cluster connections support {ccs}. All remote
-connections connect to the configured `proxy_address`. Any desired connection
-routing to gateway or <<coordinating-node,coordinating nodes>> must
-be implemented by the intermediate proxy at this configured address.
-
 [discrete]
 [[ccs-network-delays]]
-== How {ccs} handles network delays
+=== How {ccs} handles network delays
 
 Because {ccs} involves sending requests to remote clusters, any network delays
 can impact search speed. To avoid slow searches, {ccs} offers two options for
@@ -303,9 +300,9 @@ low latency.
 +
 See <<ccs-unmin-roundtrips>> to learn how this option works.
 
-[discrete]
+[float]
 [[ccs-min-roundtrips]]
-=== Minimize network roundtrips
+==== Minimize network roundtrips
 
 Here's how {ccs} works when you minimize network roundtrips.
 
@@ -329,9 +326,9 @@ final results in the {ccs} response.
 +
 image:images/ccs/ccs-min-roundtrip-client-response.svg[]
 
-[discrete]
+[float]
 [[ccs-unmin-roundtrips]]
-=== Don't minimize network roundtrips
+==== Don't minimize network roundtrips
 
 Here's how {ccs} works when you don't minimize network roundtrips.
 
diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc
index 07b9fd23f96dc..0d0b24f31e25d 100644
--- a/docs/reference/modules/remote-clusters.asciidoc
+++ b/docs/reference/modules/remote-clusters.asciidoc
@@ -13,31 +13,23 @@ connections to a remote cluster. This functionality is used in
 <<modules-cross-cluster-search,{ccs}>>.
 endif::[]
 
+// tag::how-remote-clusters-work[]
 Remote cluster connections work by configuring a remote cluster and connecting
-to a limited number of nodes in that remote cluster. There are two modes for
-remote cluster connections: <<sniff-mode,sniff mode>> and
-<<proxy-mode,proxy mode>>.
+only to a limited number of nodes in that remote cluster. Each remote cluster
+is referenced by a name and a list of seed nodes. When a remote cluster is
+registered, its cluster state is retrieved from one of the seed nodes and up
+to three _gateway nodes_ are selected to be connected to as part of remote
+cluster requests.
+// end::how-remote-clusters-work[]
 
 All the communication required between different clusters
 goes through the <<modules-transport,transport layer>>. Remote cluster
 connections consist of uni-directional connections from the coordinating
-node to the remote remote connections.
-
-[float]
-[[sniff-mode]]
-=== Sniff mode
-
-In sniff mode, a cluster is created using a name and a list of seed nodes. When
-a remote cluster is registered, its cluster state is retrieved from one of the
-seed nodes and up to three _gateway nodes_ are selected as part of remote
-cluster requests. This mode requires that the gateway node's publish addresses
-are accessible by the local cluster.
-
-Sniff mode is the default connection mode.
+node to the selected remote _gateway nodes_ only.
 
 [float]
 [[gateway-nodes-selection]]
-==== Gateway nodes selection
+=== Gateway nodes selection
 
 The _gateway nodes_ selection depends on the following criteria:
 
@@ -70,24 +62,9 @@ communicate with 6.7. The matrix below summarizes compatibility as described abo
 (see <<remote-cluster-settings>>), though such tagged nodes still have
 to satisfy the two above requirements.
 
-[float]
-[[proxy-mode]]
-=== Proxy mode
-
-In proxy mode, a cluster is created using a name and a single proxy address. When
-a remote cluster is registered, a configurable number of socket connections are
-opened to the proxy address. The proxy is required to route those connections to
-the remote cluster. Proxy mode does not require remote cluster nodes to have
-accessible publish addresses.
-
-The proxy mode is not the default connection mode and must be configured. Similar
-to the sniff <<gateway-nodes-selection,gateway nodes>>, the remote
-connections are subject to the same version compatibility rules as
-<<rolling-upgrades>>.
-
 [float]
 [[configuring-remote-clusters]]
-==== Configuring remote clusters
+=== Configuring remote clusters
 
 You can configure remote clusters globally by using
 <<cluster-update-settings,cluster settings>>, which you can update dynamically.
@@ -106,32 +83,23 @@ cluster:
         cluster_one: <1>
             seeds: 127.0.0.1:9300 <2>
             transport.ping_schedule: 30s <3>
-        cluster_two: <1>
-            mode: sniff <4>
-            seeds: 127.0.0.1:9301 <2>
-            transport.compress: true <5>
-            skip_unavailable: true <6>
-        cluster_three: <1>
-            mode: proxy <4>
-            proxy_address: 127.0.0.1:9302 <7>
+        cluster_two: 
+            seeds: 127.0.0.1:9301
+            transport.compress: true <4>
+            skip_unavailable: true <5>
 
 --------------------------------
-<1> `cluster_one`, `cluster_two`, and `cluster_three` are arbitrary _cluster aliases_
-representing the connection to each cluster. These names are subsequently used to
-distinguish between local and remote indices.
+<1> `cluster_one` and `cluster_two` are arbitrary _cluster aliases_ representing
+the connection to each cluster. These names are subsequently used to distinguish
+between local and remote indices.
 <2> The hostname and <<modules-transport,transport>> port (default: 9300) of a
 seed node in the remote cluster.
 <3> A keep-alive ping is configured for `cluster_one`.
-<4> The configured connection mode. By default, this is <<sniff-mode,`sniff`>>, so
-the mode is implicit for `cluster_one`. However, it can be explicitly configured
-as demonstrated by `cluster_two` and must be explicitly configured for
-<<proxy-mode,proxy mode>> as demonstrated by `cluster_three`.
-<5> Compression is explicitly enabled for requests to `cluster_two`.
-<6> Disconnected remote clusters are optional for `cluster_two`.
-<7> The address for the proxy endpoint used to connect to `cluster_three`.
+<4> Compression is explicitly enabled for requests to `cluster_two`.
+<5> Disconnected remote clusters are optional for `cluster_two`.
 
 For more information about the optional transport settings, see
-<<modules-transport>>.
+<<modules-transport>>. 
 
 
 If you use <<cluster-update-settings,cluster settings>>, the remote clusters
@@ -151,7 +119,6 @@ PUT _cluster/settings
           "transport.ping_schedule": "30s"
         },
         "cluster_two": {
-          "mode": "sniff",
           "seeds": [
             "127.0.0.1:9301"
           ],
@@ -159,8 +126,9 @@ PUT _cluster/settings
           "skip_unavailable": true
         },
         "cluster_three": {
-          "mode": "proxy",
-          "proxy_address": "127.0.0.1:9302"
+          "seeds": [
+            "127.0.0.1:9302"
+          ]
         }
       }
     }
@@ -171,8 +139,7 @@ PUT _cluster/settings
 // TEST[s/127.0.0.1:9300/\${transport_host}/]
 
 You can dynamically update the compression and ping schedule settings. However,
-you must re-include seeds or `proxy_address` in the settings update request.
-For example:
+you must re-include seeds in the settings update request. For example:
 
 [source,console]
 --------------------------------
@@ -188,16 +155,10 @@ PUT _cluster/settings
           "transport.ping_schedule": "60s"
         },
         "cluster_two": {
-          "mode": "sniff",
           "seeds": [
             "127.0.0.1:9301"
           ],
           "transport.compress": false
-        },
-        "cluster_three": {
-          "mode": "proxy",
-          "proxy_address": "127.0.0.1:9302",
-          "transport.compress": true
         }
       }
     }
@@ -210,7 +171,7 @@ NOTE: When the compression or ping schedule settings change, all the existing
 node connections must close and re-open, which can cause in-flight requests to
 fail.
 
-A remote cluster can be deleted from the cluster settings by setting its settings to `null` :
+A remote cluster can be deleted from the cluster settings by setting its seeds and optional settings to `null` :
 
 [source,console]
 --------------------------------
@@ -220,7 +181,6 @@ PUT _cluster/settings
     "cluster": {
       "remote": {
         "cluster_two": { <1>
-          "mode": null,
           "seeds": null,
           "skip_unavailable": null,
           "transport": {
@@ -239,21 +199,25 @@ PUT _cluster/settings
 
 [float]
 [[remote-cluster-settings]]
-=== Remote cluster settings for all modes
+=== Remote cluster settings
 
-These settings apply to both <<sniff-mode,sniff mode>> and
-<<proxy-mode,proxy mode>>. <<remote-cluster-sniff-settings,Sniff mode settings>>
-and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
+`cluster.remote.connections_per_cluster`::
 
-`cluster.remote.<cluster_alias>.mode`::
-  The mode used for a remote cluster connection. The only supported modes are
-  `sniff` and `proxy`.
+  The number of gateway nodes to connect to per remote cluster. The default is
+  `3`.
 
 `cluster.remote.initial_connect_timeout`::
 
   The time to wait for remote connections to be established when the node
   starts. The default is `30s`.
 
+`cluster.remote.node.attr`::
+
+  A node attribute to filter out nodes that are eligible as a gateway node in
+  the remote cluster. For instance a node can have a node attribute
+  `node.attr.gateway: true` such that only nodes with this attribute will be
+  connected to if `cluster.remote.node.attr` is set to `gateway`.
+
 `cluster.remote.connect`::
 
   By default, any node in the cluster can act as a cross-cluster client and
@@ -262,7 +226,7 @@ and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
   remote clusters. Remote cluster requests must be sent to a node that is
   allowed to act as a cross-cluster client.
 
-`cluster.remote.<cluster_alias>.skip_unavailable`::
+`cluster.remote.${cluster_alias}.skip_unavailable`::
 
   Per cluster boolean setting that allows to skip specific clusters when no
   nodes belonging to them are available and they are the targetof a remote
@@ -270,7 +234,7 @@ and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
   by default, but they can selectively be made optional by setting this setting
   to `true`.
 
-`cluster.remote.<cluster_alias>.transport.ping_schedule`::
+`cluster.remote.${cluster_alias}.transport.ping_schedule`::
 
   Sets the time interval between regular application-level ping messages that
   are sent to ensure that transport connections to nodes belonging to remote
@@ -279,7 +243,7 @@ and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
   are sent according to the global `transport.ping_schedule` setting, which
   defaults to `-1` meaning that pings are not sent.
 
-`cluster.remote.<cluster_alias>.transport.compress`::
+`cluster.remote.${cluster_alias}.transport.compress`::
 
   Per cluster boolean setting that enables you to configure compression for
   requests to a specific remote cluster. This setting impacts only requests
@@ -287,48 +251,6 @@ and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
   Elasticsearch compresses the response. If unset, the global
   `transport.compress` is used as the fallback setting.
 
-[float]
-[[remote-cluster-sniff-settings]]
-=== Remote cluster settings for sniff mode
-
-`cluster.remote.<cluster_alias>.seeds`::
-
-  The list of seed nodes used to sniff the remote cluster state.
-
-`cluster.remote.<cluster_alias>.node_connections`::
-
-  The number of gateway nodes to connect to for this remote cluster. The default
-  is `3`.
-
-`cluster.remote.node.attr`::
-
-  A node attribute to filter out nodes that are eligible as a gateway node in
-  the remote cluster. For instance a node can have a node attribute
-  `node.attr.gateway: true` such that only nodes with this attribute will be
-  connected to if `cluster.remote.node.attr` is set to `gateway`.
-
-[float]
-[[remote-cluster-proxy-settings]]
-=== Remote cluster settings for proxy mode
-
-`cluster.remote.<cluster_alias>.proxy_address`::
-
-  The address used for all remote connections.
-
-`cluster.remote.<cluster_alias>.proxy_socket_connections`::
-
-  The number of socket connections to open per remote cluster. The default is
-  `18`.
-
-[role="xpack"]
-`cluster.remote.<cluster_alias>.server_name`::
-
-  An optional hostname string which is sent in the `server_name` field of
-  the TLS Server Name Indication extension if
-  <<configuring-tls,TLS is enabled>>. The TLS transport will fail to open
-  remote connections if this field is not a valid hostname as defined by the
-  TLS SNI specification.
-
 [float]
 [[retrieve-remote-clusters-info]]
 === Retrieving remote clusters info
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 9ab175ee4d8e4..9e6fe18d754d0 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -370,13 +370,3 @@ See <<slm-api-start>>.
 === Stop {slm} API
 
 See <<slm-api-stop>>.
-
-[role="exclude",id="ccs-works"]
-=== How {ccs} works
-
-See <<ccs-gateway-seed-nodes>> and <<ccs-min-roundtrips>>.
-
-[role="exclude",id="async-search"]
-=== Asynchronous search
-
-coming::[7.x]
diff --git a/docs/reference/search/request/stored-fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc
index bca21a9da4fc6..c1f0feaed96d3 100644
--- a/docs/reference/search/request/stored-fields.asciidoc
+++ b/docs/reference/search/request/stored-fields.asciidoc
@@ -40,7 +40,12 @@ If the requested fields are not stored (`store` mapping set to `false`), they wi
 
 Stored field values fetched from the document itself are always returned as an array. On the contrary, metadata fields like `_routing` are never returned as an array.
 
-Also only leaf fields can be returned via the `stored_fields` option. If an object field is specified, it will be ignored.
+Also only leaf fields can be returned via the `field` option. So object fields can't be returned and such requests
+will fail.
+
+Script fields can also be automatically detected and used as fields, so
+things like `_source.obj1.field1` can be used, though not recommended, as
+`obj1.field1` will work as well.
 
 NOTE: On its own, `stored_fields` cannot be used to load fields in nested
 objects -- if a field contains a nested object in its path, then no data will
diff --git a/docs/reference/sql/language/data-types.asciidoc b/docs/reference/sql/language/data-types.asciidoc
index 2c799dc7276eb..811bb1ac6a47f 100644
--- a/docs/reference/sql/language/data-types.asciidoc
+++ b/docs/reference/sql/language/data-types.asciidoc
@@ -13,22 +13,21 @@ s|SQL precision
 
 4+h| Core types
 
-| <<null-value, `null`>>                   | null            | NULL        | 0
-| <<boolean, `boolean`>>                   | boolean         | BOOLEAN     | 1
-| <<number, `byte`>>                       | byte            | TINYINT     | 3
-| <<number, `short`>>                      | short           | SMALLINT    | 5
-| <<number, `integer`>>                    | integer         | INTEGER     | 10
-| <<number, `long`>>                       | long            | BIGINT      | 19
-| <<number, `double`>>                     | double          | DOUBLE      | 15
-| <<number, `float`>>                      | float           | REAL        | 7
-| <<number, `half_float`>>                 | half_float      | FLOAT       | 3
-| <<number, `scaled_float`>>               | scaled_float    | DOUBLE      | 15
-| <<keyword, `keyword`>>                   | keyword         | VARCHAR     | 32,766
-| <<constant-keyword, `constant_keyword`>> | constant_keyword| VARCHAR     | 32,766
-| <<text, `text`>>                         | text            | VARCHAR     | 2,147,483,647
-| <<binary, `binary`>>                     | binary          | VARBINARY   | 2,147,483,647
-| <<date, `date`>>                         | datetime        | TIMESTAMP   | 29
-| <<ip, `ip`>>                             | ip              | VARCHAR     | 39
+| <<null-value, `null`>>      | null          | NULL        | 0
+| <<boolean, `boolean`>>      | boolean       | BOOLEAN     | 1
+| <<number, `byte`>>          | byte          | TINYINT     | 3
+| <<number, `short`>>         | short         | SMALLINT    | 5
+| <<number, `integer`>>       | integer       | INTEGER     | 10
+| <<number, `long`>>          | long          | BIGINT      | 19
+| <<number, `double`>>        | double        | DOUBLE      | 15
+| <<number, `float`>>         | float         | REAL        | 7
+| <<number, `half_float`>>    | half_float    | FLOAT       | 3
+| <<number, `scaled_float`>>  | scaled_float  | DOUBLE      | 15
+| <<keyword, `keyword`>>      | keyword       | VARCHAR     | 32,766
+| <<text, `text`>>            | text          | VARCHAR     | 2,147,483,647
+| <<binary, `binary`>>        | binary        | VARBINARY   | 2,147,483,647
+| <<date, `date`>>            | datetime      | TIMESTAMP   | 29
+| <<ip, `ip`>>                | ip            | VARCHAR     | 39
 
 4+h| Complex types
 
diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc
index 1175fba036bb4..e23a6fb2fd54b 100644
--- a/docs/reference/transform/apis/get-transform-stats.asciidoc
+++ b/docs/reference/transform/apis/get-transform-stats.asciidoc
@@ -104,10 +104,6 @@ upper bound of data that is included in the checkpoint.
 `checkpointing`.`next`.`timestamp_millis`::::
 (date) The timestamp of the checkpoint, which indicates when the checkpoint was
 created.
-`checkpointing`.`operations_behind`:::
-(integer) The number of operations that have occurred on the source index but
-have not been applied to the destination index yet. A high number can indicate
-that the {transform} is failing to keep up. 
 
 `id`::
 (string)
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
index 8303cef383f44..e0ea0d511e277 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
@@ -47,31 +47,6 @@ public abstract class AbstractObjectParser<Value, Context>
     public abstract <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Context, T> parser, ParseField parseField,
             ValueType type);
 
-    /**
-     * Declares a single named object.
-     *
-     * <pre>
-     * <code>
-     * {
-     *   "object_name": {
-     *     "instance_name": { "field1": "value1", ... }
-     *     }
-     *   }
-     * }
-     * </code>
-     * </pre>
-     *
-     * @param consumer
-     *            sets the value once it has been parsed
-     * @param namedObjectParser
-     *            parses the named object
-     * @param parseField
-     *            the field to parse
-     */
-    public abstract <T> void declareNamedObject(BiConsumer<Value, T> consumer, NamedObjectParser<T, Context> namedObjectParser,
-                                                 ParseField parseField);
-
-
     /**
      * Declares named objects in the style of aggregations. These are named
      * inside and object like this:
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
index 52bfcc651609e..099701e688136 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
@@ -206,15 +206,16 @@ public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Contex
             throw new IllegalArgumentException("[type] is required");
         }
 
-        if (isConstructorArg(consumer)) {
+        if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
             /*
-             * Build a new consumer directly against the object parser that
-             * triggers the "constructor arg just arrived behavior" of the
-             * parser. Conveniently, we can close over the position of the
-             * constructor in the argument list so we don't need to do any fancy
+             * Constructor arguments are detected by these "marker" consumers. It keeps the API looking clean even if it is a bit sleezy. We
+             * then build a new consumer directly against the object parser that triggers the "constructor arg just arrived behavior" of the
+             * parser. Conveniently, we can close over the position of the constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = addConstructorArg(consumer, parseField);
+            int position = constructorArgInfos.size();
+            boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
+            constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
             objectParser.declareField((target, v) -> target.constructorArg(position, v), parser, parseField, type);
         } else {
             numberOfFields += 1;
@@ -222,39 +223,9 @@ public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Contex
         }
     }
 
-    @Override
-    public <T> void declareNamedObject(BiConsumer<Value, T> consumer, NamedObjectParser<T, Context> namedObjectParser,
-                                                ParseField parseField) {
-        if (consumer == null) {
-            throw new IllegalArgumentException("[consumer] is required");
-        }
-        if (namedObjectParser == null) {
-            throw new IllegalArgumentException("[parser] is required");
-        }
-        if (parseField == null) {
-            throw new IllegalArgumentException("[parseField] is required");
-        }
-
-        if (isConstructorArg(consumer)) {
-            /*
-             * Build a new consumer directly against the object parser that
-             * triggers the "constructor arg just arrived behavior" of the
-             * parser. Conveniently, we can close over the position of the
-             * constructor in the argument list so we don't need to do any fancy
-             * or expensive lookups whenever the constructor args come in.
-             */
-            int position = addConstructorArg(consumer, parseField);
-            objectParser.declareNamedObject((target, v) -> target.constructorArg(position, v), namedObjectParser, parseField);
-        } else {
-            numberOfFields += 1;
-            objectParser.declareNamedObject(queueingConsumer(consumer, parseField), namedObjectParser, parseField);
-        }
-    }
-
     @Override
     public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedObjectParser<T, Context> namedObjectParser,
             ParseField parseField) {
-
         if (consumer == null) {
             throw new IllegalArgumentException("[consumer] is required");
         }
@@ -265,15 +236,19 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
             throw new IllegalArgumentException("[parseField] is required");
         }
 
-        if (isConstructorArg(consumer)) {
+        if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
             /*
-             * Build a new consumer directly against the object parser that
+             * Constructor arguments are detected by this "marker" consumer. It
+             * keeps the API looking clean even if it is a bit sleezy. We then
+             * build a new consumer directly against the object parser that
              * triggers the "constructor arg just arrived behavior" of the
              * parser. Conveniently, we can close over the position of the
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = addConstructorArg(consumer, parseField);
+            int position = constructorArgInfos.size();
+            boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
+            constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
             objectParser.declareNamedObjects((target, v) -> target.constructorArg(position, v), namedObjectParser, parseField);
         } else {
             numberOfFields += 1;
@@ -297,15 +272,19 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
             throw new IllegalArgumentException("[parseField] is required");
         }
 
-        if (isConstructorArg(consumer)) {
+        if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
             /*
-             * Build a new consumer directly against the object parser that
+             * Constructor arguments are detected by this "marker" consumer. It
+             * keeps the API looking clean even if it is a bit sleezy. We then
+             * build a new consumer directly against the object parser that
              * triggers the "constructor arg just arrived behavior" of the
              * parser. Conveniently, we can close over the position of the
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = addConstructorArg(consumer, parseField);
+            int position = constructorArgInfos.size();
+            boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
+            constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
             objectParser.declareNamedObjects((target, v) -> target.constructorArg(position, v), namedObjectParser,
                     wrapOrderedModeCallBack(orderedModeCallback), parseField);
         } else {
@@ -315,27 +294,6 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
         }
     }
 
-    /**
-     * Constructor arguments are detected by this "marker" consumer. It
-     * keeps the API looking clean even if it is a bit sleezy.
-     */
-    private boolean isConstructorArg(BiConsumer<?, ?> consumer) {
-        return consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER;
-    }
-
-    /**
-     * Add a constructor argument
-     * @param consumer Either {@link #REQUIRED_CONSTRUCTOR_ARG_MARKER} or {@link #REQUIRED_CONSTRUCTOR_ARG_MARKER}
-     * @param parseField Parse field
-     * @return The argument position
-     */
-    private int addConstructorArg(BiConsumer<?, ?> consumer, ParseField parseField) {
-        int position = constructorArgInfos.size();
-        boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
-        constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
-        return position;
-    }
-    
     @Override
     public String getName() {
         return objectParser.getName();
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index b11f6afd4bb26..7526fc7bd6922 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -394,32 +394,6 @@ public <T> void declareObjectOrDefault(BiConsumer<Value, T> consumer, BiFunction
         }, field, ValueType.OBJECT_OR_BOOLEAN);
     }
 
-    @Override
-    public <T> void declareNamedObject(BiConsumer<Value, T> consumer, NamedObjectParser<T, Context> namedObjectParser,
-                                       ParseField field) {
-
-        BiFunction<XContentParser, Context, T> objectParser = (XContentParser p, Context c) -> {
-            try {
-                XContentParser.Token token = p.nextToken();
-                assert token == XContentParser.Token.FIELD_NAME;
-                String name = p.currentName();
-                try {
-                    T namedObject = namedObjectParser.parse(p, c, name);
-                    // consume the end object token
-                    token = p.nextToken();
-                    assert token == XContentParser.Token.END_OBJECT;
-                    return namedObject;
-                } catch (Exception e) {
-                    throw new XContentParseException(p.getTokenLocation(), "[" + field + "] failed to parse field [" + name + "]", e);
-                }
-            } catch (IOException e) {
-                throw new XContentParseException(p.getTokenLocation(), "[" + field + "] error while parsing named object", e);
-            }
-        };
-
-        declareField((XContentParser p, Value v, Context c) -> consumer.accept(v, objectParser.apply(p, c)), field, ValueType.OBJECT);
-    }
-
     @Override
     public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedObjectParser<T, Context> namedObjectParser,
             Consumer<Value> orderedModeCallback, ParseField field) {
@@ -429,7 +403,7 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
                 throw new XContentParseException(p.getTokenLocation(), "[" + field + "] can be a single object with any number of "
                         + "fields or an array where each entry is an object with a single field");
             }
-            // This messy exception nesting has the nice side effect of telling the user which field failed to parse
+            // This messy exception nesting has the nice side effect of telling the use which field failed to parse
             try {
                 String name = p.currentName();
                 try {
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index c99d1b10d6a4d..3e8d5dee8614f 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -500,70 +500,55 @@ public void setString_or_null(String string_or_null) {
     }
 
     public void testParseNamedObject() throws IOException {
-        XContentParser parser = createParser(JsonXContent.jsonXContent,
-                "{\"named\": { \"a\": {\"foo\" : 11} }, \"bar\": \"baz\"}");
-        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
-        assertEquals("a", h.named.name);
-        assertEquals(11, h.named.foo);
-        assertEquals("baz", h.bar);
-    }
-
-    public void testParseNamedObjectUnexpectedArray() throws IOException {
-        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ \"a\": {\"foo\" : 11} }]");
-        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
-        assertThat(e.getMessage(), containsString("[named_object_holder] named doesn't support values of type: START_ARRAY"));
-    }
-
-    public void testParseNamedObjects() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": { \"a\": {} }}");
-        NamedObjectsHolder h = NamedObjectsHolder.PARSER.apply(parser, null);
+        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertFalse(h.namedSuppliedInOrder);
     }
 
-    public void testParseNamedObjectsInOrder() throws IOException {
+    public void testParseNamedObjectInOrder() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {\"a\": {}} ] }");
-        NamedObjectsHolder h = NamedObjectsHolder.PARSER.apply(parser, null);
+        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertTrue(h.namedSuppliedInOrder);
     }
 
-    public void testParseNamedObjectsTwoFieldsInArray() throws IOException {
+    public void testParseNamedObjectTwoFieldsInArray() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {\"a\": {}, \"b\": {}}]}");
-        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectsHolder.PARSER.apply(parser, null));
-        assertThat(e.getMessage(), containsString("[named_objects_holder] failed to parse field [named]"));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
+        assertThat(e.getMessage(), containsString("[named_object_holder] failed to parse field [named]"));
         assertThat(e.getCause().getMessage(),
                 containsString("[named] can be a single object with any number of fields " +
                     "or an array where each entry is an object with a single field"));
     }
 
-    public void testParseNamedObjectsNoFieldsInArray() throws IOException {
+    public void testParseNamedObjectNoFieldsInArray() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {} ]}");
-        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectsHolder.PARSER.apply(parser, null));
-        assertThat(e.getMessage(), containsString("[named_objects_holder] failed to parse field [named]"));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
+        assertThat(e.getMessage(), containsString("[named_object_holder] failed to parse field [named]"));
         assertThat(e.getCause().getMessage(),
                 containsString("[named] can be a single object with any number of fields " +
                     "or an array where each entry is an object with a single field"));
     }
 
-    public void testParseNamedObjectsJunkInArray() throws IOException {
+    public void testParseNamedObjectJunkInArray() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ \"junk\" ] }");
-        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectsHolder.PARSER.apply(parser, null));
-        assertThat(e.getMessage(), containsString("[named_objects_holder] failed to parse field [named]"));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
+        assertThat(e.getMessage(), containsString("[named_object_holder] failed to parse field [named]"));
         assertThat(e.getCause().getMessage(),
                 containsString("[named] can be a single object with any number of fields " +
                     "or an array where each entry is an object with a single field"));
     }
 
-    public void testParseNamedObjectsInOrderNotSupported() throws IOException {
+    public void testParseNamedObjectInOrderNotSupported() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {\"a\": {}} ] }");
 
         // Create our own parser for this test so we can disable support for the "ordered" mode specified by the array above
-        ObjectParser<NamedObjectsHolder, Void> objectParser = new ObjectParser<>("named_object_holder",
-                NamedObjectsHolder::new);
-        objectParser.declareNamedObjects(NamedObjectsHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
+        ObjectParser<NamedObjectHolder, Void> objectParser = new ObjectParser<>("named_object_holder",
+                NamedObjectHolder::new);
+        objectParser.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
 
         // Now firing the xml through it fails
         XContentParseException e = expectThrows(XContentParseException.class, () -> objectParser.apply(parser, null));
@@ -728,7 +713,7 @@ public void testNoopDeclareField() throws IOException {
         assertEquals("parser for [noop] did not end on END_ARRAY", e.getMessage());
     }
 
-    public void testNoopDeclareObjectArray() {
+    public void testNoopDeclareObjectArray() throws IOException {
         ObjectParser<AtomicReference<String>, Void> parser = new ObjectParser<>("noopy", AtomicReference::new);
         parser.declareString(AtomicReference::set, new ParseField("body"));
         parser.declareObjectArray((a,b) -> {}, (p, c) -> null, new ParseField("noop"));
@@ -743,33 +728,11 @@ public void testNoopDeclareObjectArray() {
         assertEquals("expected value but got [FIELD_NAME]", sneakyError.getCause().getMessage());
     }
 
-    // singular
     static class NamedObjectHolder {
         public static final ObjectParser<NamedObjectHolder, Void> PARSER = new ObjectParser<>("named_object_holder",
                 NamedObjectHolder::new);
         static {
-            PARSER.declareNamedObject(NamedObjectHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
-            PARSER.declareString(NamedObjectHolder::setBar, new ParseField("bar"));
-        }
-
-        private NamedObject named;
-        private String bar;
-
-        public void setNamed(NamedObject named) {
-            this.named = named;
-        }
-
-        public void setBar(String bar) {
-            this.bar = bar;
-        }
-    }
-
-    // plural
-    static class NamedObjectsHolder {
-        public static final ObjectParser<NamedObjectsHolder, Void> PARSER = new ObjectParser<>("named_objects_holder",
-                NamedObjectsHolder::new);
-        static {
-            PARSER.declareNamedObjects(NamedObjectsHolder::setNamed, NamedObject.PARSER, NamedObjectsHolder::keepNamedInOrder,
+            PARSER.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, NamedObjectHolder::keepNamedInOrder,
                     new ParseField("named"));
         }
 
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index 6789f96d883b8..ce9dc7d574c12 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -28,9 +28,8 @@ dependencies {
   // Upgrade to 2.10.0 or higher when jackson-core gets upgraded to 2.9.x. Blocked by #27032
   compile('com.maxmind.geoip2:geoip2:2.9.0')
   // geoip2 dependencies:
-  // do not hardcode this to the version in version.properties, it needs to be upgraded separately with geoip2
-  compile("com.fasterxml.jackson.core:jackson-annotations:2.8.11")
-  compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.6")
+  compile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
+  compile("com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}")
   compile('com.maxmind.db:maxmind-db:1.2.2')
 
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
new file mode 100644
index 0000000000000..5203969bcf5c0
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
@@ -0,0 +1 @@
+596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 19c383c96f0a0..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1219c9aca51a37ea3e22cf88ad2e8745d1a6e02f
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..d892076210e87
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+48cb44f1dc8d3368d70581ffdbeab98ac5f5167f
\ No newline at end of file
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
index dfc0f3d0228b2..fe8e13d7ee239 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
@@ -72,6 +72,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
             searchTemplateRequest = SearchTemplateRequest.fromXContent(parser);
         }
         searchTemplateRequest.setRequest(searchRequest);
+        RestSearchAction.checkRestTotalHits(request, searchRequest);
 
         return channel -> client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel));
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
index 586deae9b7309..a78cfbd2b6eec 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
@@ -32,13 +32,11 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -112,27 +110,8 @@ static SearchRequest convert(SearchTemplateRequest searchTemplateRequest, Search
             builder.parseXContent(parser, false);
             builder.explain(searchTemplateRequest.isExplain());
             builder.profile(searchTemplateRequest.isProfile());
-            checkRestTotalHitsAsInt(searchRequest, builder);
             searchRequest.source(builder);
         }
         return searchRequest;
     }
-
-    private static void checkRestTotalHitsAsInt(SearchRequest searchRequest, SearchSourceBuilder searchSourceBuilder) {
-        if (searchRequest.source() == null) {
-            searchRequest.source(new SearchSourceBuilder());
-        }
-        Integer trackTotalHitsUpTo = searchRequest.source().trackTotalHitsUpTo();
-        // trackTotalHitsUpTo is set to Integer.MAX_VALUE when `rest_total_hits_as_int` is true
-        if (trackTotalHitsUpTo != null) {
-            if (searchSourceBuilder.trackTotalHitsUpTo() == null) {
-                // trackTotalHitsUpTo should be set here, ensure that we can get an accurate total hits count
-                searchSourceBuilder.trackTotalHitsUpTo(trackTotalHitsUpTo);
-            } else if (searchSourceBuilder.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_ACCURATE
-                && searchSourceBuilder.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
-                throw new IllegalArgumentException("[" + RestSearchAction.TOTAL_HITS_AS_INT_PARAM + "] cannot be used " +
-                    "if the tracking of total hits is not accurate, got " + searchSourceBuilder.trackTotalHitsUpTo());
-            }
-        }
-    }
 }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
index a9d3c2da68617..f4d4f3a97e5e0 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
@@ -40,19 +40,6 @@
 
   - match: { hits.total: 2 }
 
----
-"Test with invalid track_total_hits":
-
-  - do:
-      catch:              bad_request
-      search_template:
-        rest_total_hits_as_int: true
-        body: { "source" : { "query": { "match_{{template}}": {} }, "track_total_hits": "{{trackTotalHits}}" }, "params" : { "template" : "all", "trackTotalHits" : 1 } }
-
-  - match: { status: 400 }
-  - match: { error.type: illegal_argument_exception }
-  - match: { error.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" }
-
 ---
 "Missing template search request":
 
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
index e92e10b9ad276..8f72583b61d7c 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
@@ -94,11 +94,6 @@ setup:
           - source: '{"query": {"{{query_type}}": {} }' # Unknown query type
             params:
               query_type: "unknown"
-          # Search 4 has an unsupported track_total_hits
-          - index: index_*
-          - source: '{"query": {"match_all": {} }, "track_total_hits": "{{trackTotalHits}}" }'
-            params:
-              trackTotalHits: 1
 
   - match:  { responses.0.hits.total:     2  }
   - match:  { responses.1.error.root_cause.0.type: json_e_o_f_exception }
@@ -106,9 +101,6 @@ setup:
   - match:  { responses.2.hits.total:     1  }
   - match:  { responses.3.error.root_cause.0.type: parsing_exception }
   - match:  { responses.3.error.root_cause.0.reason: "/unknown.query.\\[unknown\\]/" }
-  - match:  { responses.4.error.root_cause.0.type: illegal_argument_exception }
-  - match:  { responses.4.error.root_cause.0.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" }
-
 
 ---
 "Multi-search template with invalid request":
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
index f6d0442c0110d..828e26e56b49d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
@@ -27,39 +27,37 @@
  */
 public enum Operation {
 
-    MUL     ( "*"   , "multiplication"         ),
-    DIV     ( "/"   , "division"               ),
-    REM     ( "%"   , "remainder"              ),
-    ADD     ( "+"   , "addition"               ),
-    SUB     ( "-"   , "subtraction"            ),
-    FIND    ( "=~"  , "find"                   ),
-    MATCH   ( "==~" , "match"                  ),
-    LSH     ( "<<"  , "left shift"             ),
-    RSH     ( ">>"  , "right shift"            ),
-    USH     ( ">>>" , "unsigned shift"         ),
-    BWNOT   ( "~"   , "bitwise not"            ),
-    BWAND   ( "&"   , "bitwise and"            ),
-    XOR     ( "^"   , "bitwise xor"            ),
-    BWOR    ( "|"   , "boolean or"             ),
-    NOT     ( "!"   , "boolean not"            ),
-    AND     ( "&&"  , "boolean and"            ),
-    OR      ( "||"  , "boolean or"             ),
-    LT      ( "<"   , "less than"              ),
-    LTE     ( "<="  , "less than or equals"    ),
-    GT      ( ">"   , "greater than"           ),
-    GTE     ( ">="  , "greater than or equals" ),
-    EQ      ( "=="  , "equals"                 ),
-    EQR     ( "===" , "reference equals"       ),
-    NE      ( "!="  , "not equals"             ),
-    NER     ( "!==" , "reference not equals"   ),
-    INCR    ( "++"  , "increment"              ),
-    DECR    ( "--"  , "decrement"              );
+    MUL     ( "*"   ),
+    DIV     ( "/"   ),
+    REM     ( "%"   ),
+    ADD     ( "+"   ),
+    SUB     ( "-"   ),
+    FIND    ( "=~"  ),
+    MATCH   ( "==~" ),
+    LSH     ( "<<"  ),
+    RSH     ( ">>"  ),
+    USH     ( ">>>" ),
+    BWNOT   ( "~"   ),
+    BWAND   ( "&"   ),
+    XOR     ( "^"   ),
+    BWOR    ( "|"   ),
+    NOT     ( "!"   ),
+    AND     ( "&&"  ),
+    OR      ( "||"  ),
+    LT      ( "<"   ),
+    LTE     ( "<="  ),
+    GT      ( ">"   ),
+    GTE     ( ">="  ),
+    EQ      ( "=="  ),
+    EQR     ( "===" ),
+    NE      ( "!="  ),
+    NER     ( "!==" ),
+    INCR    ( "++"  ),
+    DECR    ( "--"  );
 
     public final String symbol;
-    public final String name;
 
-    Operation(final String symbol, final String name) {
+    Operation(final String symbol) {
         this.symbol = symbol;
-        this.name = name;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 189cb94dcd70b..8d35968386f32 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -35,52 +35,6 @@
  */
 public abstract class AExpression extends ANode {
 
-    public static class Input {
-
-        /**
-         * Set to false when an expression will not be read from such as
-         * a basic assignment.  Note this variable is always set by the parent
-         * as input.
-         */
-        boolean read = true;
-
-        /**
-         * Set to the expected type this node needs to be.  Note this variable
-         * is always set by the parent as input and should never be read from.
-         */
-        Class<?> expected = null;
-
-        /**
-         * Set by {@link EExplicit} if a cast made on an expression node should be
-         * explicit.
-         */
-        boolean explicit = false;
-
-        /**
-         * Set to true if a cast is allowed to boxed/unboxed.  This is used
-         * for method arguments because casting may be required.
-         */
-        boolean internal = false;
-    }
-
-    public static class Output {
-
-        /**
-         * Set to true when an expression can be considered a stand alone
-         * statement.  Used to prevent extraneous bytecode. This is always
-         * set by the node as output.
-         */
-        boolean statement = false;
-
-        /**
-         * Set to the actual type this node is.  Note this variable is always
-         * set by the node as output and should only be read from outside of the
-         * node itself.  <b>Also, actual can always be read after a cast is
-         * called on this node to get the type of the node after the cast.</b>
-         */
-        Class<?> actual = null;
-    }
-
     /**
      * Prefix is the predecessor to this node in a variable chain.
      * This is used to analyze and write variable chains in a
@@ -90,14 +44,50 @@ public static class Output {
      */
     AExpression prefix;
 
-    // TODO: remove placeholders once analysis and write are combined into build
-    // TODO: https://github.com/elastic/elasticsearch/issues/53561
-    // This are used to support the transition from a mutable to immutable state.
+    /**
+     * Set to false when an expression will not be read from such as
+     * a basic assignment.  Note this variable is always set by the parent
+     * as input.
+     */
+    boolean read = true;
+
+    /**
+     * Set to true when an expression can be considered a stand alone
+     * statement.  Used to prevent extraneous bytecode. This is always
+     * set by the node as output.
+     */
+    boolean statement = false;
+
+    /**
+     * Set to the expected type this node needs to be.  Note this variable
+     * is always set by the parent as input and should never be read from.
+     */
+    Class<?> expected = null;
+
+    /**
+     * Set to the actual type this node is.  Note this variable is always
+     * set by the node as output and should only be read from outside of the
+     * node itself.  <b>Also, actual can always be read after a cast is
+     * called on this node to get the type of the node after the cast.</b>
+     */
+    Class<?> actual = null;
+
+    /**
+     * Set by {@link EExplicit} if a cast made on an expression node should be
+     * explicit.
+     */
+    boolean explicit = false;
+
+    /**
+     * Set to true if a cast is allowed to boxed/unboxed.  This is used
+     * for method arguments because casting may be required.
+     */
+    boolean internal = false;
+
+    // This is used to support the transition from a mutable to immutable state.
     // Currently, the IR tree is built during the user tree "write" phase, so
-    // these are stored on the node to set during the "semantic" phase and then
+    // this is stored on the node to set during the "semantic" phase and then
     // use during the "write" phase.
-    Input input = null;
-    Output output = null;
     PainlessCast cast = null;
 
     /**
@@ -121,9 +111,7 @@ public static class Output {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        throw new UnsupportedOperationException();
-    }
+    abstract void analyze(ScriptRoot scriptRoot, Scope scope);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
@@ -131,7 +119,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
     abstract ExpressionNode write(ClassNode classNode);
 
     void cast() {
-        cast = AnalyzerCaster.getLegalCast(location, output.actual, input.expected, input.explicit, input.internal);
+        cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit, internal);
     }
 
     ExpressionNode cast(ExpressionNode expressionNode) {
@@ -141,7 +129,7 @@ ExpressionNode cast(ExpressionNode expressionNode) {
 
         CastNode castNode = new CastNode();
         castNode.setLocation(location);
-        castNode.setExpressionType(cast.targetType);
+        castNode.setExpressionType(expected);
         castNode.setCast(cast);
         castNode.setChildNode(expressionNode);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index d8d12d8a14698..3ad2b0b8a0800 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -30,77 +30,66 @@
  */
 public abstract class AStatement extends ANode {
 
-    public static class Input {
-
-        /**
-         * Set to true when the final statement in an {@link SClass} is reached.
-         * Used to determine whether or not an auto-return is necessary.
-         */
-        boolean lastSource = false;
-
-        /**
-         * Set to true when a loop begins.  Used by {@link SBlock} to help determine
-         * when the final statement of a loop is reached.
-         */
-        boolean beginLoop = false;
-
-        /**
-         * Set to true when inside a loop.  Used by {@link SBreak} and {@link SContinue}
-         * to determine if a break/continue statement is legal.
-         */
-        boolean inLoop = false;
-
-        /**
-         * Set to true when on the last statement of a loop.  Used by {@link SContinue}
-         * to prevent extraneous continue statements.
-         */
-        boolean lastLoop = false;
-    }
+    /**
+     * Set to true when the final statement in an {@link SClass} is reached.
+     * Used to determine whether or not an auto-return is necessary.
+     */
+    boolean lastSource = false;
 
-    public static class Output {
-
-        /**
-         * Set to true if a statement would cause the method to exit.  Used to
-         * determine whether or not an auto-return is necessary.
-         */
-        boolean methodEscape = false;
-
-        /**
-         * Set to true if a statement would cause a loop to exit.  Used to
-         * prevent unreachable statements.
-         */
-        boolean loopEscape = false;
-
-        /**
-         * Set to true if all current paths escape from the current {@link SBlock}.
-         * Used during the analysis phase to prevent unreachable statements and
-         * the writing phase to prevent extraneous bytecode gotos from being written.
-         */
-        boolean allEscape = false;
-
-        /**
-         * Set to true if any continue statement occurs in a loop.  Used to prevent
-         * unnecessary infinite loops.
-         */
-        boolean anyContinue = false;
-
-        /**
-         * Set to true if any break statement occurs in a loop.  Used to prevent
-         * extraneous loops.
-         */
-        boolean anyBreak = false;
-
-        /**
-         * Set to the approximate number of statements in a loop block to prevent
-         * infinite loops during runtime.
-         */
-        int statementCount = 0;
-    }
+    /**
+     * Set to true when a loop begins.  Used by {@link SBlock} to help determine
+     * when the final statement of a loop is reached.
+     */
+    boolean beginLoop = false;
+
+    /**
+     * Set to true when inside a loop.  Used by {@link SBreak} and {@link SContinue}
+     * to determine if a break/continue statement is legal.
+     */
+    boolean inLoop = false;
+
+    /**
+     * Set to true when on the last statement of a loop.  Used by {@link SContinue}
+     * to prevent extraneous continue statements.
+     */
+    boolean lastLoop = false;
+
+    /**
+     * Set to true if a statement would cause the method to exit.  Used to
+     * determine whether or not an auto-return is necessary.
+     */
+    boolean methodEscape = false;
 
-    // TODO: remove placeholders once analysis and write are combined into build
-    // TODO: https://github.com/elastic/elasticsearch/issues/53561
-    Input input;
-    Output output;
+    /**
+     * Set to true if a statement would cause a loop to exit.  Used to
+     * prevent unreachable statements.
+     */
+    boolean loopEscape = false;
+
+    /**
+     * Set to true if all current paths escape from the current {@link SBlock}.
+     * Used during the analysis phase to prevent unreachable statements and
+     * the writing phase to prevent extraneous bytecode gotos from being written.
+     */
+    boolean allEscape = false;
+
+    /**
+     * Set to true if any continue statement occurs in a loop.  Used to prevent
+     * unnecessary infinite loops.
+     */
+    boolean anyContinue = false;
+
+    /**
+     * Set to true if any break statement occurs in a loop.  Used to prevent
+     * extraneous loops.
+     */
+    boolean anyBreak = false;
+
+    /**
+     * Set to the approximate number of statements in a loop block to prevent
+     * infinite loops during runtime.
+     */
+    int statementCount = 0;
 
     /**
      * Standard constructor with location used for error tracking.
@@ -112,9 +101,7 @@ public static class Output {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        throw new UnsupportedOperationException();
-    }
+    abstract void analyze(ScriptRoot scriptRoot, Scope scope);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
index cc7262f670a94..7f5beda267959 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Scope;
-import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
 
@@ -30,14 +28,11 @@
  */
 abstract class AStoreable extends AExpression {
 
-    public static class Input extends AExpression.Input {
-
-        /**
-         * Set to true when this node is an lhs-expression and will be storing
-         * a value from an rhs-expression.
-         */
-        boolean write = false;
-    }
+    /**
+     * Set to true when this node is an lhs-expression and will be storing
+     * a value from an rhs-expression.
+     */
+    boolean write = false;
 
     /**
      * Standard constructor with location used for error tracking.
@@ -57,10 +52,6 @@ public static class Input extends AExpression.Input {
         this.prefix = Objects.requireNonNull(prefix);
     }
 
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        throw new UnsupportedOperationException();
-    }
-
     /**
      * Returns true if this node or a sub-node of this node can be optimized with
      * rhs actual type to avoid an unnecessary cast.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
index e995d637520c7..4de8bdcb9a1e1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
@@ -62,24 +62,32 @@ public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        analyzeLHS(scriptRoot, scope);
+        analyzeIncrDecr();
 
-        Output leftOutput;
-        Output rightOutput;
+        if (operation != null) {
+            analyzeCompound(scriptRoot, scope);
+        } else if (rhs != null) {
+            analyzeSimple(scriptRoot, scope);
+        } else {
+            throw new IllegalStateException("Illegal tree structure.");
+        }
+    }
 
+    private void analyzeLHS(ScriptRoot scriptRoot, Scope scope) {
         if (lhs instanceof AStoreable) {
             AStoreable lhs = (AStoreable)this.lhs;
-            AStoreable.Input leftInput = new AStoreable.Input();
 
-            leftInput.read = input.read;
-            leftInput.write = true;
-            leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
+            lhs.read = read;
+            lhs.write = true;
+            lhs.analyze(scriptRoot, scope);
         } else {
             throw new IllegalArgumentException("Left-hand side cannot be assigned a value.");
         }
+    }
 
+    private void analyzeIncrDecr() {
         if (pre && post) {
             throw createError(new IllegalStateException("Illegal tree structure."));
         } else if (pre || post) {
@@ -88,11 +96,11 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
 
             if (operation == Operation.INCR) {
-                if (leftOutput.actual == double.class) {
+                if (lhs.actual == double.class) {
                     rhs = new EConstant(location, 1D);
-                } else if (leftOutput.actual == float.class) {
+                } else if (lhs.actual == float.class) {
                     rhs = new EConstant(location, 1F);
-                } else if (leftOutput.actual == long.class) {
+                } else if (lhs.actual == long.class) {
                     rhs = new EConstant(location, 1L);
                 } else {
                     rhs = new EConstant(location, 1);
@@ -100,11 +108,11 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
                 operation = Operation.ADD;
             } else if (operation == Operation.DECR) {
-                if (leftOutput.actual == double.class) {
+                if (lhs.actual == double.class) {
                     rhs = new EConstant(location, 1D);
-                } else if (leftOutput.actual == float.class) {
+                } else if (lhs.actual == float.class) {
                     rhs = new EConstant(location, 1F);
-                } else if (leftOutput.actual == long.class) {
+                } else if (lhs.actual == long.class) {
                     rhs = new EConstant(location, 1L);
                 } else {
                     rhs = new EConstant(location, 1);
@@ -115,105 +123,103 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
+    }
 
-        if (operation != null) {
-            rightOutput = rhs.analyze(scriptRoot, scope, new Input());
-            boolean shift = false;
-
-            if (operation == Operation.MUL) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
-            } else if (operation == Operation.DIV) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
-            } else if (operation == Operation.REM) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
-            } else if (operation == Operation.ADD) {
-                promote = AnalyzerCaster.promoteAdd(leftOutput.actual, rightOutput.actual);
-            } else if (operation == Operation.SUB) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
-            } else if (operation == Operation.LSH) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
-                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
-                shift = true;
-            } else if (operation == Operation.RSH) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
-                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
-                shift = true;
-            } else if (operation == Operation.USH) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
-                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
-                shift = true;
-            } else if (operation == Operation.BWAND) {
-                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
-            } else if (operation == Operation.XOR) {
-                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
-            } else if (operation == Operation.BWOR) {
-                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
-            } else {
-                throw createError(new IllegalStateException("Illegal tree structure."));
-            }
+    private void analyzeCompound(ScriptRoot scriptRoot, Scope scope) {
+        rhs.analyze(scriptRoot, scope);
+        boolean shift = false;
+
+        if (operation == Operation.MUL) {
+            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
+        } else if (operation == Operation.DIV) {
+            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
+        } else if (operation == Operation.REM) {
+            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
+        } else if (operation == Operation.ADD) {
+            promote = AnalyzerCaster.promoteAdd(lhs.actual, rhs.actual);
+        } else if (operation == Operation.SUB) {
+            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
+        } else if (operation == Operation.LSH) {
+            promote = AnalyzerCaster.promoteNumeric(lhs.actual, false);
+            shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false);
+            shift = true;
+        } else if (operation == Operation.RSH) {
+            promote = AnalyzerCaster.promoteNumeric(lhs.actual, false);
+            shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false);
+            shift = true;
+        } else if (operation == Operation.USH) {
+            promote = AnalyzerCaster.promoteNumeric(lhs.actual, false);
+            shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false);
+            shift = true;
+        } else if (operation == Operation.BWAND) {
+            promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual);
+        } else if (operation == Operation.XOR) {
+            promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual);
+        } else if (operation == Operation.BWOR) {
+            promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual);
+        } else {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
 
-            if (promote == null || (shift && shiftDistance == null)) {
-                throw createError(new ClassCastException("Cannot apply compound assignment " +
-                        "[" + operation.symbol + "=] to types [" + leftOutput.actual + "] and [" + rightOutput.actual + "]."));
-            }
+        if (promote == null || (shift && shiftDistance == null)) {
+            throw createError(new ClassCastException("Cannot apply compound assignment " +
+                "[" + operation.symbol + "=] to types [" + lhs.actual + "] and [" + rhs.actual + "]."));
+        }
 
-            cat = operation == Operation.ADD && promote == String.class;
+        cat = operation == Operation.ADD && promote == String.class;
 
-            if (cat) {
-                if (rhs instanceof EBinary && ((EBinary)rhs).operation == Operation.ADD && rightOutput.actual == String.class) {
-                    ((EBinary)rhs).cat = true;
-                }
+        if (cat) {
+            if (rhs instanceof EBinary && ((EBinary)rhs).operation == Operation.ADD && rhs.actual == String.class) {
+                ((EBinary)rhs).cat = true;
             }
 
-            if (shift) {
-                if (promote == def.class) {
-                    // shifts are promoted independently, but for the def type, we need object.
-                    rhs.input.expected = promote;
-                } else if (shiftDistance == long.class) {
-                    rhs.input.expected = int.class;
-                    rhs.input.explicit = true;
-                } else {
-                    rhs.input.expected = shiftDistance;
-                }
+            rhs.expected = rhs.actual;
+        } else if (shift) {
+            if (promote == def.class) {
+                // shifts are promoted independently, but for the def type, we need object.
+                rhs.expected = promote;
+            } else if (shiftDistance == long.class) {
+                rhs.expected = int.class;
+                rhs.explicit = true;
             } else {
-                rhs.input.expected = promote;
+                rhs.expected = shiftDistance;
             }
+        } else {
+            rhs.expected = promote;
+        }
 
-            rhs.cast();
-
-            there = AnalyzerCaster.getLegalCast(location, leftOutput.actual, promote, false, false);
-            back = AnalyzerCaster.getLegalCast(location, promote, leftOutput.actual, true, false);
+        rhs.cast();
 
+        there = AnalyzerCaster.getLegalCast(location, lhs.actual, promote, false, false);
+        back = AnalyzerCaster.getLegalCast(location, promote, lhs.actual, true, false);
 
-        } else if (rhs != null) {
-            AStoreable lhs = (AStoreable)this.lhs;
+        this.statement = true;
+        this.actual = read ? lhs.actual : void.class;
+    }
 
-            // If the lhs node is a def optimized node we update the actual type to remove the need for a cast.
-            if (lhs.isDefOptimized()) {
-                rightOutput = rhs.analyze(scriptRoot, scope, new Input());
+    private void analyzeSimple(ScriptRoot scriptRoot, Scope scope) {
+        AStoreable lhs = (AStoreable)this.lhs;
 
-                if (rightOutput.actual == void.class) {
-                    throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment."));
-                }
+        // If the lhs node is a def optimized node we update the actual type to remove the need for a cast.
+        if (lhs.isDefOptimized()) {
+            rhs.analyze(scriptRoot, scope);
 
-                rhs.input.expected = rightOutput.actual;
-                lhs.updateActual(rightOutput.actual);
-            // Otherwise, we must adapt the rhs type to the lhs type with a cast.
-            } else {
-                Input rightInput = new Input();
-                rightInput.expected = leftOutput.actual;
-                rhs.analyze(scriptRoot, scope, rightInput);
+            if (rhs.actual == void.class) {
+                throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment."));
             }
 
-            rhs.cast();
+            rhs.expected = rhs.actual;
+            lhs.updateActual(rhs.actual);
+        // Otherwise, we must adapt the rhs type to the lhs type with a cast.
         } else {
-            throw new IllegalStateException("Illegal tree structure.");
+            rhs.expected = lhs.actual;
+            rhs.analyze(scriptRoot, scope);
         }
 
-        output.statement = true;
-        output.actual = input.read ? leftOutput.actual : void.class;
+        rhs.cast();
 
-        return output;
+        this.statement = true;
+        this.actual = read ? lhs.actual : void.class;
     }
 
     /**
@@ -230,12 +236,12 @@ AssignmentNode write(ClassNode classNode) {
         assignmentNode.setRightNode(rhs.cast(rhs.write(classNode)));
 
         assignmentNode.setLocation(location);
-        assignmentNode.setExpressionType(output.actual);
+        assignmentNode.setExpressionType(actual);
         assignmentNode.setCompoundType(promote);
         assignmentNode.setPre(pre);
         assignmentNode.setPost(post);
         assignmentNode.setOperation(operation);
-        assignmentNode.setRead(input.read);
+        assignmentNode.setRead(read);
         assignmentNode.setCat(cat);
         assignmentNode.setThere(there);
         assignmentNode.setBack(back);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 5a65d74d4158c..b5584c4b19360 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -55,87 +55,415 @@ public EBinary(Location location, Operation operation, AExpression left, AExpres
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        originallyExplicit = explicit;
 
-        originallyExplicit = input.explicit;
+        if (operation == Operation.MUL) {
+            analyzeMul(scriptRoot, scope);
+        } else if (operation == Operation.DIV) {
+            analyzeDiv(scriptRoot, scope);
+        } else if (operation == Operation.REM) {
+            analyzeRem(scriptRoot, scope);
+        } else if (operation == Operation.ADD) {
+            analyzeAdd(scriptRoot, scope);
+        } else if (operation == Operation.SUB) {
+            analyzeSub(scriptRoot, scope);
+        } else if (operation == Operation.FIND) {
+            analyzeRegexOp(scriptRoot, scope);
+        } else if (operation == Operation.MATCH) {
+            analyzeRegexOp(scriptRoot, scope);
+        } else if (operation == Operation.LSH) {
+            analyzeLSH(scriptRoot, scope);
+        } else if (operation == Operation.RSH) {
+            analyzeRSH(scriptRoot, scope);
+        } else if (operation == Operation.USH) {
+            analyzeUSH(scriptRoot, scope);
+        } else if (operation == Operation.BWAND) {
+            analyzeBWAnd(scriptRoot, scope);
+        } else if (operation == Operation.XOR) {
+            analyzeXor(scriptRoot, scope);
+        } else if (operation == Operation.BWOR) {
+            analyzeBWOr(scriptRoot, scope);
+        } else {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+    }
+
+    private void analyzeMul(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply multiply [*] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote;
+
+        if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeDiv(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply divide [/] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote;
+
+        if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeRem(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply remainder [%] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote;
+
+        if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeAdd(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply add [+] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote;
+
+        if (promote == String.class) {
+            left.expected = left.actual;
+
+            if (left instanceof EBinary && ((EBinary)left).operation == Operation.ADD && left.actual == String.class) {
+                ((EBinary)left).cat = true;
+            }
+
+            right.expected = right.actual;
+
+            if (right instanceof EBinary && ((EBinary)right).operation == Operation.ADD && right.actual == String.class) {
+                ((EBinary)right).cat = true;
+            }
+        } else if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeSub(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply subtract [-] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote;
+
+        if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left.cast();
+        right.cast();
+    }
 
-        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
-        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
+    private void analyzeRegexOp(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
 
-        if (operation == Operation.FIND || operation == Operation.MATCH) {
-            left.input.expected = String.class;
-            right.input.expected = Pattern.class;
-            promote = boolean.class;
-            output.actual = boolean.class;
+        left.expected = String.class;
+        right.expected = Pattern.class;
+
+        left.cast();
+        right.cast();
+
+        promote = boolean.class;
+        actual = boolean.class;
+    }
+
+    private void analyzeLSH(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        Class<?> lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Class<?> rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
+
+        if (lhspromote == null || rhspromote == null) {
+            throw createError(new ClassCastException("Cannot apply left shift [<<] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote = lhspromote;
+        shiftDistance = rhspromote;
+
+        if (lhspromote == def.class || rhspromote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
+            }
         } else {
-            if (operation == Operation.MUL || operation == Operation.DIV || operation == Operation.REM || operation == Operation.SUB) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
-            } else if (operation == Operation.ADD) {
-                promote = AnalyzerCaster.promoteAdd(leftOutput.actual, rightOutput.actual);
-            } else if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
-                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
-
-                if (shiftDistance == null) {
-                    promote = null;
-                }
-            } else if (operation == Operation.BWOR || operation == Operation.BWAND) {
-                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, false);
-            } else if (operation == Operation.XOR) {
-                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
+            left.expected = lhspromote;
+
+            if (rhspromote == long.class) {
+                right.expected = int.class;
+                right.explicit = true;
             } else {
-                throw createError(new IllegalStateException("unexpected binary operation [" + operation.name + "]"));
+                right.expected = rhspromote;
             }
+        }
+
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeRSH(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
 
-            if (promote == null) {
-                throw createError(new ClassCastException("cannot apply the " + operation.name + " operator " +
-                        "[" + operation.symbol + "] to the types " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(leftOutput.actual) + "] and " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
+        Class<?> lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Class<?> rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
+
+        if (lhspromote == null || rhspromote == null) {
+            throw createError(new ClassCastException("Cannot apply right shift [>>] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote = lhspromote;
+        shiftDistance = rhspromote;
+
+        if (lhspromote == def.class || rhspromote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
             }
+        } else {
+            left.expected = lhspromote;
 
-            output.actual = promote;
+            if (rhspromote == long.class) {
+                right.expected = int.class;
+                right.explicit = true;
+            } else {
+                right.expected = rhspromote;
+            }
+        }
 
-            if (operation == Operation.ADD && promote == String.class) {
-                left.input.expected = leftOutput.actual;
-                right.input.expected = rightOutput.actual;
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeUSH(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
 
-                if (left instanceof EBinary && ((EBinary) left).operation == Operation.ADD && leftOutput.actual == String.class) {
-                    ((EBinary) left).cat = true;
-                }
+        Class<?> lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Class<?> rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
 
-                if (right instanceof EBinary && ((EBinary) right).operation == Operation.ADD && rightOutput.actual == String.class) {
-                    ((EBinary) right).cat = true;
-                }
-            } else if (promote == def.class || shiftDistance != null && shiftDistance == def.class) {
-                left.input.expected = leftOutput.actual;
-                right.input.expected = rightOutput.actual;
+        actual = promote = lhspromote;
+        shiftDistance = rhspromote;
+
+        if (lhspromote == null || rhspromote == null) {
+            throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        if (lhspromote == def.class || rhspromote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = lhspromote;
 
-                if (input.expected != null) {
-                    output.actual = input.expected;
-                }
+            if (rhspromote == long.class) {
+                right.expected = int.class;
+                right.explicit = true;
             } else {
-                left.input.expected = promote;
+                right.expected = rhspromote;
+            }
+        }
+
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeBWAnd(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply and [&] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote;
+
+        if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left.cast();
+        right.cast();
+    }
+
+    private void analyzeXor(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply xor [^] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
 
-                if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
-                    if (shiftDistance == long.class) {
-                        right.input.expected = int.class;
-                        right.input.explicit = true;
-                    } else {
-                        right.input.expected = shiftDistance;
-                    }
-                } else {
-                    right.input.expected = promote;
-                }
+        actual = promote;
+
+        if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
             }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
         }
 
         left.cast();
         right.cast();
+    }
+
+    private void analyzeBWOr(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
-        return output;
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply or [|] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        actual = promote;
+
+        if (promote == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left.cast();
+        right.cast();
     }
 
     @Override
@@ -146,7 +474,7 @@ BinaryMathNode write(ClassNode classNode) {
         binaryMathNode.setRightNode(right.cast(right.write(classNode)));
 
         binaryMathNode.setLocation(location);
-        binaryMathNode.setExpressionType(output.actual);
+        binaryMathNode.setExpressionType(actual);
         binaryMathNode.setBinaryType(promote);
         binaryMathNode.setShiftType(shiftDistance);
         binaryMathNode.setOperation(operation);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index f2acae9299f20..c7538ffa7a887 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -46,23 +46,16 @@ public EBool(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        Input leftInput = new Input();
-        leftInput.expected = boolean.class;
-        left.analyze(scriptRoot, scope, leftInput);
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        left.expected = boolean.class;
+        left.analyze(scriptRoot, scope);
         left.cast();
 
-        Input rightInput = new Input();
-        rightInput.expected = boolean.class;
-        right.analyze(scriptRoot, scope, rightInput);
+        right.expected = boolean.class;
+        right.analyze(scriptRoot, scope);
         right.cast();
 
-        output.actual = boolean.class;
-
-        return output;
+        actual = boolean.class;
     }
 
     @Override
@@ -73,7 +66,7 @@ BooleanNode write(ClassNode classNode) {
         booleanNode.setRightNode(right.cast(right.write(classNode)));
 
         booleanNode.setLocation(location);
-        booleanNode.setExpressionType(output.actual);
+        booleanNode.setExpressionType(actual);
         booleanNode.setOperation(operation);
 
         return booleanNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 6e78fb972008c..10d941fdce90f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -40,24 +40,19 @@ public EBoolean(Location location, boolean constant) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
         }
 
-        output.actual = boolean.class;
-
-        return output;
+        actual = boolean.class;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(output.actual);
+        constantNode.setExpressionType(actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index ad4dce45d4d7a..fa2e1fce72c4f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -58,10 +58,7 @@ public ECallLocal(Location location, String name, List<AExpression> arguments) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         localFunction = scriptRoot.getFunctionTable().getFunction(name, arguments.size());
 
         // user cannot call internal functions, reset to null if an internal function is found
@@ -115,21 +112,21 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         if (localFunction != null) {
             typeParameters = new ArrayList<>(localFunction.getTypeParameters());
-            output.actual = localFunction.getReturnType();
+            actual = localFunction.getReturnType();
         } else if (importedMethod != null) {
             scriptRoot.markNonDeterministic(importedMethod.annotations.containsKey(NonDeterministicAnnotation.class));
             typeParameters = new ArrayList<>(importedMethod.typeParameters);
-            output.actual = importedMethod.returnType;
+            actual = importedMethod.returnType;
         } else if (classBinding != null) {
             scriptRoot.markNonDeterministic(classBinding.annotations.containsKey(NonDeterministicAnnotation.class));
             typeParameters = new ArrayList<>(classBinding.typeParameters);
-            output.actual = classBinding.returnType;
+            actual = classBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("class_binding");
             scriptRoot.getClassNode().addField(new SField(location,
                     Modifier.PRIVATE, bindingName, classBinding.javaConstructor.getDeclaringClass()));
         } else if (instanceBinding != null) {
             typeParameters = new ArrayList<>(instanceBinding.typeParameters);
-            output.actual = instanceBinding.returnType;
+            actual = instanceBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("instance_binding");
             scriptRoot.getClassNode().addField(new SField(location, Modifier.STATIC | Modifier.PUBLIC,
                     bindingName, instanceBinding.targetInstance.getClass()));
@@ -144,16 +141,13 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = typeParameters.get(argument + classBindingOffset);
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = typeParameters.get(argument + classBindingOffset);
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
 
-        output.statement = true;
-
-        return output;
+        statement = true;
     }
 
     @Override
@@ -165,7 +159,7 @@ MemberCallNode write(ClassNode classNode) {
         }
 
         memberCallNode.setLocation(location);
-        memberCallNode.setExpressionType(output.actual);
+        memberCallNode.setExpressionType(actual);
         memberCallNode.setLocalFunction(localFunction);
         memberCallNode.setImportedMethod(importedMethod);
         memberCallNode.setClassBinding(classBinding);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index fa8b25cd861f9..adf67370e3b03 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -51,12 +51,9 @@ public ECapturingFunctionRef(Location location, String variable, String call) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         captured = scope.getVariable(location, variable);
-        if (input.expected == null) {
+        if (expected == null) {
             if (captured.getType() == def.class) {
                 // dynamic implementation
                 defPointer = "D" + variable + "." + call + ",1";
@@ -64,18 +61,16 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 // typed implementation
                 defPointer = "S" + captured.getCanonicalTypeName() + "." + call + ",1";
             }
-            output.actual = String.class;
+            actual = String.class;
         } else {
             defPointer = null;
             // static case
             if (captured.getType() != def.class) {
                 ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(), location,
-                        input.expected, captured.getCanonicalTypeName(), call, 1);
+                        expected, captured.getCanonicalTypeName(), call, 1);
             }
-            output.actual = input.expected;
+            actual = expected;
         }
-
-        return output;
     }
 
     @Override
@@ -83,7 +78,7 @@ CapturingFuncRefNode write(ClassNode classNode) {
         CapturingFuncRefNode capturingFuncRefNode = new CapturingFuncRefNode();
 
         capturingFuncRefNode.setLocation(location);
-        capturingFuncRefNode.setExpressionType(output.actual);
+        capturingFuncRefNode.setExpressionType(actual);
         capturingFuncRefNode.setCapturedName(captured.getName());
         capturingFuncRefNode.setName(call);
         capturingFuncRefNode.setPointer(defPointer);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 32555c18625f7..512acee938a78 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -51,47 +51,240 @@ public EComp(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (operation == Operation.EQ) {
+            analyzeEq(scriptRoot, scope);
+        } else if (operation == Operation.EQR) {
+            analyzeEqR(scriptRoot, scope);
+        } else if (operation == Operation.NE) {
+            analyzeNE(scriptRoot, scope);
+        } else if (operation == Operation.NER) {
+            analyzeNER(scriptRoot, scope);
+        } else if (operation == Operation.GTE) {
+            analyzeGTE(scriptRoot, scope);
+        } else if (operation == Operation.GT) {
+            analyzeGT(scriptRoot, scope);
+        } else if (operation == Operation.LTE) {
+            analyzeLTE(scriptRoot, scope);
+        } else if (operation == Operation.LT) {
+            analyzeLT(scriptRoot, scope);
+        } else {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+    }
 
-        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
-        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
+    private void analyzeEq(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+
+        if (promotedType == null) {
+            throw createError(new ClassCastException("Cannot apply equals [==] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
 
-        if (operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER) {
-            promotedType = AnalyzerCaster.promoteEquality(leftOutput.actual, rightOutput.actual);
-        } else if (operation == Operation.GT || operation == Operation.GTE || operation == Operation.LT || operation == Operation.LTE) {
-            promotedType = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
+        if (promotedType == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
         } else {
-            throw createError(new IllegalStateException("unexpected binary operation [" + operation.name + "]"));
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
+
+        if (left instanceof ENull && right instanceof ENull) {
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
         }
 
+        left.cast();
+        right.cast();
+
+        actual = boolean.class;
+    }
+
+    private void analyzeEqR(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+
         if (promotedType == null) {
-            throw createError(new ClassCastException("cannot apply the " + operation.name + " operator " +
-                    "[" + operation.symbol + "] to the types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(leftOutput.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
+            throw createError(new ClassCastException("Cannot apply reference equals [===] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        left.expected = promotedType;
+        right.expected = promotedType;
+
+        if (left instanceof ENull && right instanceof ENull) {
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
         }
 
-        if (operation != Operation.EQR && operation != Operation.NER && promotedType == def.class) {
-            left.input.expected = leftOutput.actual;
-            right.input.expected = rightOutput.actual;
+        left.cast();
+        right.cast();
+
+        actual = boolean.class;
+    }
+
+    private void analyzeNE(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+
+        if (promotedType == null) {
+            throw createError(new ClassCastException("Cannot apply not equals [!=] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        if (promotedType == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
         } else {
-            left.input.expected = promotedType;
-            right.input.expected = promotedType;
+            left.expected = promotedType;
+            right.expected = promotedType;
         }
 
-        if ((operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER)
-                && left instanceof ENull && right instanceof ENull) {
-            throw createError(new IllegalArgumentException("extraneous comparison of [null] constants"));
+        if (left instanceof ENull && right instanceof ENull) {
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
         }
 
         left.cast();
         right.cast();
 
-        output.actual = boolean.class;
+        actual = boolean.class;
+    }
+
+    private void analyzeNER(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+
+        if (promotedType == null) {
+            throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        left.expected = promotedType;
+        right.expected = promotedType;
+
+        if (left instanceof ENull && right instanceof ENull) {
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
+        }
+
+        left.cast();
+        right.cast();
+
+        actual = boolean.class;
+    }
+
+    private void analyzeGTE(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promotedType == null) {
+            throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        if (promotedType == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
+
+        left.cast();
+        right.cast();
+
+        actual = boolean.class;
+    }
+
+    private void analyzeGT(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promotedType == null) {
+            throw createError(new ClassCastException("Cannot apply greater than [>] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        if (promotedType == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
+
+        left.cast();
+        right.cast();
+
+        actual = boolean.class;
+    }
+
+    private void analyzeLTE(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promotedType == null) {
+            throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        if (promotedType == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
+
+        left.cast();
+        right.cast();
+
+        actual = boolean.class;
+    }
+
+    private void analyzeLT(ScriptRoot scriptRoot, Scope variables) {
+        left.analyze(scriptRoot, variables);
+        right.analyze(scriptRoot, variables);
+
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+
+        if (promotedType == null) {
+            throw createError(new ClassCastException("Cannot apply less than [>=] to types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
+        }
+
+        if (promotedType == def.class) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
+
+        left.cast();
+        right.cast();
 
-        return output;
+        actual = boolean.class;
     }
 
     @Override
@@ -102,7 +295,7 @@ ComparisonNode write(ClassNode classNode) {
         comparisonNode.setRightNode(right.cast(right.write(classNode)));
 
         comparisonNode.setLocation(location);
-        comparisonNode.setExpressionType(output.actual);
+        comparisonNode.setExpressionType(actual);
         comparisonNode.setComparisonType(promotedType);
         comparisonNode.setOperation(operation);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index d523838ecf66d..c1eb8fbe510da 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -47,48 +47,38 @@ public EConditional(Location location, AExpression condition, AExpression left,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        Input conditionInput = new Input();
-        conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        condition.expected = boolean.class;
+        condition.analyze(scriptRoot, scope);
         condition.cast();
 
-        Input leftInput = new Input();
-        leftInput.expected = input.expected;
-        leftInput.explicit = input.explicit;
-        leftInput.internal = input.internal;
-
-        Input rightInput = new Input();
-        rightInput.expected = input.expected;
-        rightInput.explicit = input.explicit;
-        rightInput.internal = input.internal;
+        left.expected = expected;
+        left.explicit = explicit;
+        left.internal = internal;
+        right.expected = expected;
+        right.explicit = explicit;
+        right.internal = internal;
+        actual = expected;
 
-        output.actual = input.expected;
+        left.analyze(scriptRoot, scope);
+        right.analyze(scriptRoot, scope);
 
-        Output leftOutput = left.analyze(scriptRoot, scope, leftInput);
-        Output rightOutput = right.analyze(scriptRoot, scope, rightInput);
-
-        if (input.expected == null) {
-            Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
+        if (expected == null) {
+            Class<?> promote = AnalyzerCaster.promoteConditional(left.actual, right.actual);
 
             if (promote == null) {
-                throw createError(new ClassCastException("cannot apply the conditional operator [?:] to the types " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(leftOutput.actual) + "] and " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
+                throw createError(new ClassCastException("cannot apply a conditional operator [?:] to the types " +
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]"));
             }
 
-            left.input.expected = promote;
-            right.input.expected = promote;
-            output.actual = promote;
+            left.expected = promote;
+            right.expected = promote;
+            actual = promote;
         }
 
         left.cast();
         right.cast();
-
-        return output;
     }
 
     @Override
@@ -100,7 +90,7 @@ ConditionalNode write(ClassNode classNode) {
         conditionalNode.setConditionNode(condition.cast(condition.write(classNode)));
 
         conditionalNode.setLocation(location);
-        conditionalNode.setExpressionType(output.actual);
+        conditionalNode.setExpressionType(actual);
 
         return conditionalNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 5caa6bca92895..c02beca4ec044 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 /**
@@ -41,42 +40,35 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         if (constant instanceof String) {
-            output.actual = String.class;
+            actual = String.class;
         } else if (constant instanceof Double) {
-            output.actual = double.class;
+            actual = double.class;
         } else if (constant instanceof Float) {
-            output.actual = float.class;
+            actual = float.class;
         } else if (constant instanceof Long) {
-            output.actual = long.class;
+            actual = long.class;
         } else if (constant instanceof Integer) {
-            output.actual = int.class;
+            actual = int.class;
         } else if (constant instanceof Character) {
-            output.actual = char.class;
+            actual = char.class;
         } else if (constant instanceof Short) {
-            output.actual = short.class;
+            actual = short.class;
         } else if (constant instanceof Byte) {
-            output.actual = byte.class;
+            actual = byte.class;
         } else if (constant instanceof Boolean) {
-            output.actual = boolean.class;
+            actual = boolean.class;
         } else {
-            throw createError(new IllegalStateException("unexpected type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(constant.getClass()) + "] " +
-                    "for constant node"));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         }
-
-        return output;
     }
 
     @Override
     ConstantNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(output.actual);
+        constantNode.setExpressionType(actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 405f90b356f1a..049e52799f1f7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -44,18 +44,15 @@ public EDecimal(Location location, String value) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
         }
 
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                output.actual = float.class;
+                actual = float.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid float constant [" + value + "]."));
             }
@@ -66,20 +63,18 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
             try {
                 constant = Double.parseDouble(toParse);
-                output.actual = double.class;
+                actual = double.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid double constant [" + value + "]."));
             }
         }
-
-        return output;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(output.actual);
+        constantNode.setExpressionType(actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
index 6c771af47caf4..f9ffb019c7a7e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
@@ -44,24 +44,19 @@ public EElvis(Location location, AExpression lhs, AExpression rhs) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.expected != null && input.expected.isPrimitive()) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (expected != null && expected.isPrimitive()) {
             throw createError(new IllegalArgumentException("Elvis operator cannot return primitives"));
         }
-        Input leftInput = new Input();
-        leftInput.expected = input.expected;
-        leftInput.explicit = input.explicit;
-        leftInput.internal = input.internal;
-        Input rightInput = new Input();
-        rightInput.expected = input.expected;
-        rightInput.explicit = input.explicit;
-        rightInput.internal = input.internal;
-        output.actual = input.expected;
-        Output leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
-        Output rightOutput = rhs.analyze(scriptRoot, scope, rightInput);
+        lhs.expected = expected;
+        lhs.explicit = explicit;
+        lhs.internal = internal;
+        rhs.expected = expected;
+        rhs.explicit = explicit;
+        rhs.internal = internal;
+        actual = expected;
+        lhs.analyze(scriptRoot, scope);
+        rhs.analyze(scriptRoot, scope);
 
         if (lhs instanceof ENull) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is null."));
@@ -73,25 +68,23 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 || lhs instanceof EConstant) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is a constant."));
         }
-        if (leftOutput.actual.isPrimitive()) {
+        if (lhs.actual.isPrimitive()) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is a primitive."));
         }
         if (rhs instanceof ENull) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. RHS is null."));
         }
 
-        if (input.expected == null) {
-            Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
+        if (expected == null) {
+            Class<?> promote = AnalyzerCaster.promoteConditional(lhs.actual, rhs.actual);
 
-            lhs.input.expected = promote;
-            rhs.input.expected = promote;
-            output.actual = promote;
+            lhs.expected = promote;
+            rhs.expected = promote;
+            actual = promote;
         }
 
         lhs.cast();
         rhs.cast();
-
-        return output;
     }
 
     @Override
@@ -102,7 +95,7 @@ ElvisNode write(ClassNode classNode) {
         elvisNode.setRightNode(rhs.cast(rhs.write(classNode)));
 
         elvisNode.setLocation(location);
-        elvisNode.setExpressionType(output.actual);
+        elvisNode.setExpressionType(actual);
 
         return elvisNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index b76dcd946b3b3..d3d288bde8162 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -43,23 +43,17 @@ public EExplicit(Location location, String type, AExpression child) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
 
-        output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
-
-        if (output.actual == null) {
+        if (actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
 
-        Input childInput = new Input();
-        childInput.expected = output.actual;
-        childInput.explicit = true;
-        child.analyze(scriptRoot, scope, childInput);
+        child.expected = actual;
+        child.explicit = true;
+        child.analyze(scriptRoot, scope);
         child.cast();
-
-        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 62bb34c215590..67a2e24f0b7c6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -48,22 +48,16 @@ public EFunctionRef(Location location, String type, String call) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.expected == null) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (expected == null) {
             ref = null;
-            output.actual = String.class;
+            actual = String.class;
             defPointer = "S" + type + "." + call + ",0";
         } else {
             defPointer = null;
-            ref = FunctionRef.create(
-                    scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(), location, input.expected, type, call, 0);
-            output.actual = input.expected;
+            ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(), location, expected, type, call, 0);
+            actual = expected;
         }
-
-        return output;
     }
 
     @Override
@@ -71,7 +65,7 @@ FuncRefNode write(ClassNode classNode) {
         FuncRefNode funcRefNode = new FuncRefNode();
 
         funcRefNode.setLocation(location);
-        funcRefNode.setExpressionType(output.actual);
+        funcRefNode.setExpressionType(actual);
         funcRefNode.setFuncRef(ref);
 
         return funcRefNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index e6ee1a03602b8..01c789dd89e4f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -38,7 +38,7 @@ public final class EInstanceof extends AExpression {
     private final String type;
 
     private Class<?> resolvedType;
-    private Class<?> expressionType;
+    private Class<?> instanceType;
     private boolean primitiveExpression;
 
     public EInstanceof(Location location, AExpression expression, String type) {
@@ -48,10 +48,7 @@ public EInstanceof(Location location, AExpression expression, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         // ensure the specified type is part of the definition
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
 
@@ -64,19 +61,17 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 PainlessLookupUtility.typeToJavaType(clazz);
 
         // analyze and cast the expression
-        Output expressionOutput = expression.analyze(scriptRoot, scope, new Input());
-        expression.input.expected = expressionOutput.actual;
+        expression.analyze(scriptRoot, scope);
+        expression.expected = expression.actual;
         expression.cast();
 
         // record if the expression returns a primitive
-        primitiveExpression = expressionOutput.actual.isPrimitive();
+        primitiveExpression = expression.actual.isPrimitive();
         // map to wrapped type for primitive types
-        expressionType = expressionOutput.actual.isPrimitive() ?
-            PainlessLookupUtility.typeToBoxedType(expressionOutput.actual) : PainlessLookupUtility.typeToJavaType(clazz);
-
-        output.actual = boolean.class;
+        instanceType = expression.actual.isPrimitive() ?
+            PainlessLookupUtility.typeToBoxedType(expression.actual) : PainlessLookupUtility.typeToJavaType(clazz);
 
-        return output;
+        actual = boolean.class;
     }
 
     @Override
@@ -86,8 +81,8 @@ InstanceofNode write(ClassNode classNode) {
         instanceofNode.setChildNode(expression.cast(expression.write(classNode)));
 
         instanceofNode.setLocation(location);
-        instanceofNode.setExpressionType(output.actual);
-        instanceofNode.setInstanceType(expressionType);
+        instanceofNode.setExpressionType(actual);
+        instanceofNode.setInstanceType(instanceType);
         instanceofNode.setResolvedType(resolvedType);
         instanceofNode.setPrimitiveResult(primitiveExpression);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index a4cb19d048c62..5dbebcf554af3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -91,14 +91,11 @@ public ELambda(Location location,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         List<Class<?>> typeParameters = new ArrayList<>();
         PainlessMethod interfaceMethod;
         // inspect the target first, set interface method if we know it.
-        if (input.expected == null) {
+        if (expected == null) {
             interfaceMethod = null;
             // we don't know anything: treat as def
             returnType = def.class;
@@ -120,15 +117,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         } else {
             // we know the method statically, infer return type and any unknown/def types
-            interfaceMethod = scriptRoot.getPainlessLookup().lookupFunctionalInterfacePainlessMethod(input.expected);
+            interfaceMethod = scriptRoot.getPainlessLookup().lookupFunctionalInterfacePainlessMethod(expected);
             if (interfaceMethod == null) {
                 throw createError(new IllegalArgumentException("Cannot pass lambda to " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(input.expected) + "], not a functional interface"));
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"));
             }
             // check arity before we manipulate parameters
             if (interfaceMethod.typeParameters.size() != paramTypeStrs.size())
                 throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.javaMethod.getName() +
-                        "] in [" + PainlessLookupUtility.typeToCanonicalTypeName(input.expected) + "]");
+                        "] in [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]");
             // for method invocation, its allowed to ignore the return value
             if (interfaceMethod.returnType == void.class) {
                 returnType = def.class;
@@ -165,18 +162,17 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         if (block.statements.isEmpty()) {
             throw createError(new IllegalArgumentException("cannot generate empty lambda"));
         }
-        AStatement.Input blockInput = new AStatement.Input();
-        blockInput.lastSource = true;
-        AStatement.Output blockOutput = block.analyze(scriptRoot, lambdaScope, blockInput);
+        block.lastSource = true;
+        block.analyze(scriptRoot, lambdaScope);
+        captures = new ArrayList<>(lambdaScope.getCaptures());
 
-        if (blockOutput.methodEscape == false) {
+        if (block.methodEscape == false) {
             throw createError(new IllegalArgumentException("not all paths return a value for lambda"));
         }
 
         maxLoopCounter = scriptRoot.getCompilerSettings().getMaxLoopCounter();
 
         // prepend capture list to lambda's arguments
-        captures = new ArrayList<>(lambdaScope.getCaptures());
         this.typeParameters = new ArrayList<>(captures.size() + typeParameters.size());
         parameterNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
@@ -191,18 +187,16 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         scriptRoot.getFunctionTable().addFunction(name, returnType, this.typeParameters, true, true);
 
         // setup method reference to synthetic method
-        if (input.expected == null) {
+        if (expected == null) {
             ref = null;
-            output.actual = String.class;
+            actual = String.class;
             defPointer = "Sthis." + name + "," + captures.size();
         } else {
             defPointer = null;
             ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(),
-                    location, input.expected, "this", name, captures.size());
-            output.actual = input.expected;
+                    location, expected, "this", name, captures.size());
+            actual = expected;
         }
-
-        return output;
     }
 
     @Override
@@ -226,7 +220,7 @@ LambdaNode write(ClassNode classNode) {
         LambdaNode lambdaNode = new LambdaNode();
 
         lambdaNode.setLocation(location);
-        lambdaNode.setExpressionType(output.actual);
+        lambdaNode.setExpressionType(actual);
         lambdaNode.setFuncRef(ref);
 
         for (Variable capture : captures) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index cb49bce8a11c3..27c6d1c7897f1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -49,40 +49,34 @@ public EListInit(Location location, List<AExpression> values) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Must read from list initializer."));
         }
 
-        output.actual = ArrayList.class;
+        actual = ArrayList.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
+        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
-                    "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
+                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "add", 1);
+        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(actual, false, "add", 1);
 
         if (method == null) {
-            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", add/1] not found"));
+            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", add/1] not found"));
         }
 
         for (int index = 0; index < values.size(); ++index) {
             AExpression expression = values.get(index);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = def.class;
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = def.class;
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
-
-        return output;
     }
 
     @Override
@@ -94,7 +88,7 @@ ListInitializationNode write(ClassNode classNode) {
         }
 
         listInitializationNode.setLocation(location);
-        listInitializationNode.setExpressionType(output.actual);
+        listInitializationNode.setExpressionType(actual);
         listInitializationNode.setConstructor(constructor);
         listInitializationNode.setMethod(method);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index f846a18eb89f9..69eb4e10f8866 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -51,27 +51,24 @@ public EMapInit(Location location, List<AExpression> keys, List<AExpression> val
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Must read from map initializer."));
         }
 
-        output.actual = HashMap.class;
+        actual = HashMap.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
+        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
-                    "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
+                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "put", 2);
+        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(actual, false, "put", 2);
 
         if (method == null) {
-            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", put/2] not found"));
+            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", put/2] not found"));
         }
 
         if (keys.size() != values.size()) {
@@ -81,24 +78,20 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         for (int index = 0; index < keys.size(); ++index) {
             AExpression expression = keys.get(index);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = def.class;
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = def.class;
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
 
         for (int index = 0; index < values.size(); ++index) {
             AExpression expression = values.get(index);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = def.class;
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = def.class;
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
-
-        return output;
     }
 
     @Override
@@ -112,7 +105,7 @@ MapInitializationNode write(ClassNode classNode) {
         }
 
         mapInitializationNode.setLocation(location);
-        mapInitializationNode.setExpressionType(output.actual);
+        mapInitializationNode.setExpressionType(actual);
         mapInitializationNode.setConstructor(constructor);
         mapInitializationNode.setMethod(method);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
index 73603d79e6c89..7e2a315fc24e1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
@@ -46,11 +46,8 @@ public ENewArray(Location location, String type, List<AExpression> arguments, bo
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
              throw createError(new IllegalArgumentException("A newly created array must be read from."));
         }
 
@@ -63,16 +60,13 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = initialize ? clazz.getComponentType() : int.class;
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = initialize ? clazz.getComponentType() : int.class;
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
 
-        output.actual = clazz;
-
-        return output;
+        actual = clazz;
     }
 
     @Override
@@ -84,7 +78,7 @@ NewArrayNode write(ClassNode classNode) {
         }
 
         newArrayNode.setLocation(location);
-        newArrayNode.setExpressionType(output.actual);
+        newArrayNode.setExpressionType(actual);
         newArrayNode.setInitialize(initialize);
 
         return newArrayNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
index 0a132c12e5adb..5302f17d85907 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
@@ -48,10 +48,7 @@ public ENewArrayFunctionRef(Location location, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         SReturn code = new SReturn(location, new ENewArray(location, type, Arrays.asList(new EVariable(location, "size")), false));
         function = new SFunction(
                 location, type, scriptRoot.getNextSyntheticName("newarray"),
@@ -61,18 +58,16 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         function.analyze(scriptRoot);
         scriptRoot.getFunctionTable().addFunction(function.name, function.returnType, function.typeParameters, true, true);
 
-        if (input.expected == null) {
+        if (expected == null) {
             ref = null;
-            output.actual = String.class;
+            actual = String.class;
             defPointer = "Sthis." + function.name + ",0";
         } else {
             defPointer = null;
             ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(),
-                    location, input.expected, "this", function.name, 0);
-            output.actual = input.expected;
+                    location, expected, "this", function.name, 0);
+            actual = expected;
         }
-
-        return output;
     }
 
     @Override
@@ -82,7 +77,7 @@ NewArrayFuncRefNode write(ClassNode classNode) {
         NewArrayFuncRefNode newArrayFuncRefNode = new NewArrayFuncRefNode();
 
         newArrayFuncRefNode.setLocation(location);
-        newArrayFuncRefNode.setExpressionType(output.actual);
+        newArrayFuncRefNode.setExpressionType(actual);
         newArrayFuncRefNode.setFuncRef(ref);
 
         return newArrayFuncRefNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index dda96482a472d..0696b75696e2d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -51,21 +51,18 @@ public ENewObj(Location location, String type, List<AExpression> arguments) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
 
-        output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
-
-        if (output.actual == null) {
+        if (actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, arguments.size());
+        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(actual, arguments.size());
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
-                    "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/" + arguments.size() + "] not found"));
+                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/" + arguments.size() + "] not found"));
         }
 
         scriptRoot.markNonDeterministic(constructor.annotations.containsKey(NonDeterministicAnnotation.class));
@@ -75,23 +72,20 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         if (constructor.typeParameters.size() != arguments.size()) {
             throw createError(new IllegalArgumentException(
-                    "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(output.actual) + "] " +
+                    "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " +
                     "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "]."));
         }
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = types[argument];
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = types[argument];
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
 
-        output.statement = true;
-
-        return output;
+        statement = true;
     }
 
     @Override
@@ -103,8 +97,8 @@ NewObjectNode write(ClassNode classNode) {
         }
 
         newObjectNode.setLocation(location);
-        newObjectNode.setExpressionType(output.actual);
-        newObjectNode.setRead(input.read);
+        newObjectNode.setExpressionType(actual);
+        newObjectNode.setRead(read);
         newObjectNode.setConstructor(constructor);
 
         return newObjectNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 841bd09586bed..6d26a7de1e79f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -36,26 +36,21 @@ public ENull(Location location) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Must read from null constant."));
         }
 
-        if (input.expected != null) {
-            if (input.expected.isPrimitive()) {
+        if (expected != null) {
+            if (expected.isPrimitive()) {
                 throw createError(new IllegalArgumentException(
-                    "Cannot cast null to a primitive type [" + PainlessLookupUtility.typeToCanonicalTypeName(input.expected) + "]."));
+                    "Cannot cast null to a primitive type [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]."));
             }
 
-            output.actual = input.expected;
+            actual = expected;
         } else {
-            output.actual = Object.class;
+            actual = Object.class;
         }
-
-        return output;
     }
 
     @Override
@@ -63,7 +58,7 @@ NullNode write(ClassNode classNode) {
         NullNode nullNode = new NullNode();
 
         nullNode.setLocation(location);
-        nullNode.setExpressionType(output.actual);
+        nullNode.setExpressionType(actual);
 
         return nullNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index f7cbe681b639d..5c689f090acaf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -46,11 +46,8 @@ public ENumeric(Location location, String value, int radix) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
         }
 
@@ -61,7 +58,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
             try {
                 constant = Double.parseDouble(value.substring(0, value.length() - 1));
-                output.actual = double.class;
+                actual = double.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid double constant [" + value + "]."));
             }
@@ -72,34 +69,34 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                output.actual = float.class;
+                actual = float.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid float constant [" + value + "]."));
             }
         } else if (value.endsWith("l") || value.endsWith("L")) {
             try {
                 constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
-                output.actual = long.class;
+                actual = long.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid long constant [" + value + "]."));
             }
         } else {
             try {
-                Class<?> sort = input.expected == null ? int.class : input.expected;
+                Class<?> sort = expected == null ? int.class : expected;
                 int integer = Integer.parseInt(value, radix);
 
                 if (sort == byte.class && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) {
                     constant = (byte)integer;
-                    output.actual = byte.class;
+                    actual = byte.class;
                 } else if (sort == char.class && integer >= Character.MIN_VALUE && integer <= Character.MAX_VALUE) {
                     constant = (char)integer;
-                    output.actual = char.class;
+                    actual = char.class;
                 } else if (sort == short.class && integer >= Short.MIN_VALUE && integer <= Short.MAX_VALUE) {
                     constant = (short)integer;
-                    output.actual = short.class;
+                    actual = short.class;
                 } else {
                     constant = integer;
-                    output.actual = int.class;
+                    actual = int.class;
                 }
             } catch (NumberFormatException exception) {
                 try {
@@ -113,15 +110,13 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 throw createError(new IllegalArgumentException("Invalid int constant [" + value + "]."));
             }
         }
-
-        return output;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(output.actual);
+        constantNode.setExpressionType(actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
index 3847397407dd8..6c3527291afa9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
@@ -63,18 +63,14 @@ public ERegex(Location location, String pattern, String flagsString) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         if (scriptRoot.getCompilerSettings().areRegexesEnabled() == false) {
             throw createError(new IllegalStateException("Regexes are disabled. Set [script.painless.regex.enabled] to [true] "
                     + "in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep "
                     + "recursion and long loops."));
         }
 
-        if (input.read == false) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Regex constant may only be read [" + pattern + "]."));
         }
 
@@ -86,9 +82,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         name = scriptRoot.getNextSyntheticName("regex");
-        output.actual = Pattern.class;
-
-        return output;
+        actual = Pattern.class;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
index 0706e321e5312..9714fec7ce52c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
@@ -41,17 +41,12 @@ public EStatic(Location location, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
 
-        output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
-
-        if (output.actual == null) {
+        if (actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
-
-        return output;
     }
 
     @Override
@@ -59,7 +54,7 @@ StaticNode write(ClassNode classNode) {
         StaticNode staticNode = new StaticNode();
 
         staticNode.setLocation(location);
-        staticNode.setExpressionType(output.actual);
+        staticNode.setExpressionType(actual);
 
         return staticNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
index 5cc5b67fe511b..423ff49e521b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
@@ -42,24 +42,19 @@ public EString(Location location, String string) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.read == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!read) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
         }
 
-        output.actual = String.class;
-
-        return output;
+        actual = String.class;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(output.actual);
+        constantNode.setExpressionType(actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index e498ceb3578c8..3fe797f0352ee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -51,43 +51,88 @@ public EUnary(Location location, Operation operation, AExpression child) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        originallyExplicit = input.explicit;
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        originallyExplicit = explicit;
 
         if (operation == Operation.NOT) {
-            Input childInput = new Input();
-            childInput.expected = boolean.class;
-            child.analyze(scriptRoot, scope, childInput);
-            child.cast();
-
-            output.actual = boolean.class;
-        } else if (operation == Operation.BWNOT || operation == Operation.ADD || operation == Operation.SUB) {
-            Output childOutput = child.analyze(scriptRoot, scope, new Input());
-
-            promote = AnalyzerCaster.promoteNumeric(childOutput.actual, operation != Operation.BWNOT);
-
-            if (promote == null) {
-                throw createError(new ClassCastException("cannot apply the " + operation.name + " operator " +
-                        "[" + operation.symbol + "] to the type " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(childOutput.actual) + "]"));
-            }
-
-            child.input.expected = promote;
-            child.cast();
-
-            if (promote == def.class && input.expected != null) {
-                output.actual = input.expected;
-            } else {
-                output.actual = promote;
-            }
+            analyzeNot(scriptRoot, scope);
+        } else if (operation == Operation.BWNOT) {
+            analyzeBWNot(scriptRoot, scope);
+        } else if (operation == Operation.ADD) {
+            analyzerAdd(scriptRoot, scope);
+        } else if (operation == Operation.SUB) {
+            analyzerSub(scriptRoot, scope);
+        } else {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+    }
+
+    void analyzeNot(ScriptRoot scriptRoot, Scope variables) {
+        child.expected = boolean.class;
+        child.analyze(scriptRoot, variables);
+        child.cast();
+
+        actual = boolean.class;
+    }
+
+    void analyzeBWNot(ScriptRoot scriptRoot, Scope variables) {
+        child.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(child.actual, false);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply not [~] to type " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(child.actual) + "]."));
+        }
+
+        child.expected = promote;
+        child.cast();
+
+        if (promote == def.class && expected != null) {
+            actual = expected;
+        } else {
+            actual = promote;
+        }
+    }
+
+    void analyzerAdd(ScriptRoot scriptRoot, Scope variables) {
+        child.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(child.actual, true);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply positive [+] to type " +
+                    "[" + PainlessLookupUtility.typeToJavaType(child.actual) + "]."));
+        }
+
+        child.expected = promote;
+        child.cast();
+
+        if (promote == def.class && expected != null) {
+            actual = expected;
         } else {
-            throw createError(new IllegalStateException("unexpected unary operation [" + operation.name + "]"));
+            actual = promote;
         }
+    }
 
-        return output;
+    void analyzerSub(ScriptRoot scriptRoot, Scope variables) {
+        child.analyze(scriptRoot, variables);
+
+        promote = AnalyzerCaster.promoteNumeric(child.actual, true);
+
+        if (promote == null) {
+            throw createError(new ClassCastException("Cannot apply negative [-] to type " +
+                    "[" + PainlessLookupUtility.typeToJavaType(child.actual) + "]."));
+        }
+
+        child.expected = promote;
+        child.cast();
+
+        if (promote == def.class && expected != null) {
+            actual = expected;
+        } else {
+            actual = promote;
+        }
     }
 
     @Override
@@ -97,7 +142,7 @@ UnaryNode write(ClassNode classNode) {
         unaryMathNode.setChildNode(child.cast(child.write(classNode)));
 
         unaryMathNode.setLocation(location);
-        unaryMathNode.setExpressionType(output.actual);
+        unaryMathNode.setExpressionType(actual);
         unaryMathNode.setUnaryType(promote);
         unaryMathNode.setOperation(operation);
         unaryMathNode.setOriginallExplicit(originallyExplicit);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
index 2f1a4d6f09a72..840ec3b2b5114 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
@@ -42,30 +42,14 @@ public EVariable(Location location, String name) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
-        AStoreable.Input storeableInput = new AStoreable.Input();
-        storeableInput.read = input.read;
-        storeableInput.expected = input.expected;
-        storeableInput.explicit = input.explicit;
-        storeableInput.internal = input.internal;
-
-        return analyze(scriptRoot, scope, storeableInput);
-    }
-
-    @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         Variable variable = scope.getVariable(location, name);
 
-        if (input.write && variable.isFinal()) {
+        if (write && variable.isFinal()) {
             throw createError(new IllegalArgumentException("Variable [" + variable.getName() + "] is read-only."));
         }
 
-        output.actual = variable.getType();
-
-        return output;
+        actual = variable.getType();
     }
 
     @Override
@@ -73,7 +57,7 @@ VariableNode write(ClassNode classNode) {
         VariableNode variableNode = new VariableNode();
 
         variableNode.setLocation(location);
-        variableNode.setExpressionType(output.actual);
+        variableNode.setExpressionType(actual);
         variableNode.setName(name);
 
         return variableNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index 684b015aa835e..31246ffe56131 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -47,47 +47,30 @@ public PBrace(Location location, AExpression prefix, AExpression index) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
-        AStoreable.Input storeableInput = new AStoreable.Input();
-        storeableInput.read = input.read;
-        storeableInput.expected = input.expected;
-        storeableInput.explicit = input.explicit;
-        storeableInput.internal = input.internal;
-
-        return analyze(scriptRoot, scope, storeableInput);
-    }
-
-    @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        prefix.analyze(scriptRoot, scope);
+        prefix.expected = prefix.actual;
         prefix.cast();
 
-        if (prefixOutput.actual.isArray()) {
-            sub = new PSubBrace(location, prefixOutput.actual, index);
-        } else if (prefixOutput.actual == def.class) {
+        if (prefix.actual.isArray()) {
+            sub = new PSubBrace(location, prefix.actual, index);
+        } else if (prefix.actual == def.class) {
             sub = new PSubDefArray(location, index);
-        } else if (Map.class.isAssignableFrom(prefixOutput.actual)) {
-            sub = new PSubMapShortcut(location, prefixOutput.actual, index);
-        } else if (List.class.isAssignableFrom(prefixOutput.actual)) {
-            sub = new PSubListShortcut(location, prefixOutput.actual, index);
+        } else if (Map.class.isAssignableFrom(prefix.actual)) {
+            sub = new PSubMapShortcut(location, prefix.actual, index);
+        } else if (List.class.isAssignableFrom(prefix.actual)) {
+            sub = new PSubListShortcut(location, prefix.actual, index);
         } else {
             throw createError(new IllegalArgumentException("Illegal array access on type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
         }
 
-        Input subInput = new Input();
-        subInput.write = input.write;
-        subInput.read = input.read;
-        subInput.expected = input.expected;
-        subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
-        output.actual = subOutput.actual;
-
-        return output;
+        sub.write = write;
+        sub.read = read;
+        sub.expected = expected;
+        sub.explicit = explicit;
+        sub.analyze(scriptRoot, scope);
+        actual = sub.actual;
     }
 
     @Override
@@ -98,7 +81,7 @@ BraceNode write(ClassNode classNode) {
         braceNode.setRightNode(sub.write(classNode));
 
         braceNode.setLocation(location);
-        braceNode.setExpressionType(output.actual);
+        braceNode.setExpressionType(actual);
 
         return braceNode;
     }
@@ -111,7 +94,7 @@ boolean isDefOptimized() {
     @Override
     void updateActual(Class<?> actual) {
         sub.updateActual(actual);
-        this.output.actual = actual;
+        this.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index 4a0254ef96253..fe4ab2658302c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -53,43 +53,37 @@ public PCallInvoke(Location location, AExpression prefix, String name, boolean n
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        prefix.analyze(scriptRoot, scope);
+        prefix.expected = prefix.actual;
         prefix.cast();
 
-        if (prefixOutput.actual == def.class) {
+        if (prefix.actual == def.class) {
             sub = new PSubDefCall(location, name, arguments);
         } else {
-            PainlessMethod method = scriptRoot.getPainlessLookup().lookupPainlessMethod(
-                    prefixOutput.actual, prefix instanceof EStatic, name, arguments.size());
+            PainlessMethod method =
+                    scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, prefix instanceof EStatic, name, arguments.size());
 
             if (method == null) {
                 throw createError(new IllegalArgumentException(
-                        "method [" + typeToCanonicalTypeName(prefixOutput.actual) + ", " + name + "/" + arguments.size() + "] not found"));
+                        "method [" + typeToCanonicalTypeName(prefix.actual) + ", " + name + "/" + arguments.size() + "] not found"));
             }
 
             scriptRoot.markNonDeterministic(method.annotations.containsKey(NonDeterministicAnnotation.class));
 
-            sub = new PSubCallInvoke(location, method, prefixOutput.actual, arguments);
+            sub = new PSubCallInvoke(location, method, prefix.actual, arguments);
         }
 
         if (nullSafe) {
             sub = new PSubNullSafeCallInvoke(location, sub);
         }
 
-        Input subInput = new Input();
-        subInput.expected = input.expected;
-        subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
-        output.actual = subOutput.actual;
-
-        output.statement = true;
+        sub.expected = expected;
+        sub.explicit = explicit;
+        sub.analyze(scriptRoot, scope);
+        actual = sub.actual;
 
-        return output;
+        statement = true;
     }
 
     @Override
@@ -100,7 +94,7 @@ CallNode write(ClassNode classNode) {
         callNode.setRightNode(sub.write(classNode));
 
         callNode.setLocation(location);
-        callNode.setExpressionType(output.actual);
+        callNode.setExpressionType(actual);
 
         return callNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index 2380aa536edde..a1ad58d28570d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -53,66 +53,51 @@ public PField(Location location, AExpression prefix, boolean nullSafe, String va
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
-        AStoreable.Input storeableInput = new AStoreable.Input();
-        storeableInput.read = input.read;
-        storeableInput.expected = input.expected;
-        storeableInput.explicit = input.explicit;
-        storeableInput.internal = input.internal;
-
-        return analyze(scriptRoot, scope, storeableInput);
-    }
-
-    @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        prefix.analyze(scriptRoot, scope);
+        prefix.expected = prefix.actual;
         prefix.cast();
 
-        if (prefixOutput.actual.isArray()) {
-            sub = new PSubArrayLength(location, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), value);
-        } else if (prefixOutput.actual == def.class) {
+        if (prefix.actual.isArray()) {
+            sub = new PSubArrayLength(location, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), value);
+        } else if (prefix.actual == def.class) {
             sub = new PSubDefField(location, value);
         } else {
-            PainlessField field = scriptRoot.getPainlessLookup().lookupPainlessField(prefixOutput.actual, prefix instanceof EStatic, value);
+            PainlessField field = scriptRoot.getPainlessLookup().lookupPainlessField(prefix.actual, prefix instanceof EStatic, value);
 
             if (field == null) {
                 PainlessMethod getter;
                 PainlessMethod setter;
 
-                getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefixOutput.actual, false,
+                getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
                         "get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
 
                 if (getter == null) {
-                    getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefixOutput.actual, false,
+                    getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
                             "is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
                 }
 
-                setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefixOutput.actual, false,
+                setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
                         "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
 
                 if (getter != null || setter != null) {
-                    sub = new PSubShortcut(
-                            location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), getter, setter);
+                    sub = new PSubShortcut(location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter);
                 } else {
                     EConstant index = new EConstant(location, value);
-                    index.analyze(scriptRoot, scope, new Input());
+                    index.analyze(scriptRoot, scope);
 
-                    if (Map.class.isAssignableFrom(prefixOutput.actual)) {
-                        sub = new PSubMapShortcut(location, prefixOutput.actual, index);
+                    if (Map.class.isAssignableFrom(prefix.actual)) {
+                        sub = new PSubMapShortcut(location, prefix.actual, index);
                     }
 
-                    if (List.class.isAssignableFrom(prefixOutput.actual)) {
-                        sub = new PSubListShortcut(location, prefixOutput.actual, index);
+                    if (List.class.isAssignableFrom(prefix.actual)) {
+                        sub = new PSubListShortcut(location, prefix.actual, index);
                     }
                 }
 
                 if (sub == null) {
                     throw createError(new IllegalArgumentException(
-                            "field [" + typeToCanonicalTypeName(prefixOutput.actual) + ", " + value + "] not found"));
+                            "field [" + typeToCanonicalTypeName(prefix.actual) + ", " + value + "] not found"));
                 }
             } else {
                 sub = new PSubField(location, field);
@@ -123,15 +108,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             sub = new PSubNullSafeField(location, sub);
         }
 
-        Input subInput = new Input();
-        subInput.write = input.write;
-        subInput.read = input.read;
-        subInput.expected = input.expected;
-        subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
-        output.actual = subOutput.actual;
-
-        return output;
+        sub.write = write;
+        sub.read = read;
+        sub.expected = expected;
+        sub.explicit = explicit;
+        sub.analyze(scriptRoot, scope);
+        actual = sub.actual;
     }
 
     @Override
@@ -142,7 +124,7 @@ DotNode write(ClassNode classNode) {
         dotNode.setRightNode(sub.write(classNode));
 
         dotNode.setLocation(location);
-        dotNode.setExpressionType(output.actual);
+        dotNode.setExpressionType(actual);
 
         return dotNode;
     }
@@ -155,7 +137,7 @@ boolean isDefOptimized() {
     @Override
     void updateActual(Class<?> actual) {
         sub.updateActual(actual);
-        this.output.actual = actual;
+        this.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
index 43cdba046bd54..bbf2a086645f7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
@@ -43,21 +43,16 @@ final class PSubArrayLength extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         if ("length".equals(value)) {
-            if (input.write) {
+            if (write) {
                 throw createError(new IllegalArgumentException("Cannot write to read-only field [length] for an array."));
             }
 
-            output.actual = int.class;
+            actual = int.class;
         } else {
             throw createError(new IllegalArgumentException("Field [" + value + "] does not exist for type [" + type + "]."));
         }
-
-        return output;
     }
 
     @Override
@@ -65,7 +60,7 @@ DotSubArrayLengthNode write(ClassNode classNode) {
         DotSubArrayLengthNode dotSubArrayLengthNode = new DotSubArrayLengthNode();
 
         dotSubArrayLengthNode.setLocation(location);
-        dotSubArrayLengthNode.setExpressionType(output.actual);
+        dotSubArrayLengthNode.setExpressionType(actual);
 
         return dotSubArrayLengthNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
index 0303ce051f7f5..2ea986ac39625 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
@@ -43,18 +43,12 @@ final class PSubBrace extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
-        Input indexInput = new Input();
-        indexInput.expected = int.class;
-        index.analyze(scriptRoot, scope, indexInput);
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        index.expected = int.class;
+        index.analyze(scriptRoot, scope);
         index.cast();
 
-        output.actual = clazz.getComponentType();
-
-        return output;
+        actual = clazz.getComponentType();
     }
 
     BraceSubNode write(ClassNode classNode) {
@@ -63,7 +57,7 @@ BraceSubNode write(ClassNode classNode) {
         braceSubNode.setChildNode(index.cast(index.write(classNode)));
 
         braceSubNode.setLocation(location);
-        braceSubNode.setExpressionType(output.actual);
+        braceSubNode.setExpressionType(actual);
 
         return braceSubNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 7e509682f7d47..487621b60046a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -47,24 +47,18 @@ final class PSubCallInvoke extends AExpression {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = method.typeParameters.get(argument);
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = method.typeParameters.get(argument);
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
 
-        output.statement = true;
-        output.actual = method.returnType;
-
-        return output;
+        statement = true;
+        actual = method.returnType;
     }
 
     @Override
@@ -76,7 +70,7 @@ CallSubNode write(ClassNode classNode) {
         }
 
         callSubNode.setLocation(location);
-        callSubNode.setExpressionType(output.actual);
+        callSubNode.setExpressionType(actual);
         callSubNode.setMethod(method);
         callSubNode .setBox(box);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
index 060102ae8593f..a39a16e26be62 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
@@ -42,18 +42,13 @@ final class PSubDefArray extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
-        Output indexOutput = index.analyze(scriptRoot, scope, new Input());
-        index.input.expected = indexOutput.actual;
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        index.analyze(scriptRoot, scope);
+        index.expected = index.actual;
         index.cast();
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
-        output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
-
-        return output;
+        actual = expected == null || expected == ZonedDateTime.class || explicit ? def.class : expected;
     }
 
     @Override
@@ -63,7 +58,7 @@ BraceSubDefNode write(ClassNode classNode) {
         braceSubDefNode.setChildNode(index.cast(index.write(classNode)));
 
         braceSubDefNode.setLocation(location);
-        braceSubDefNode.setExpressionType(output.actual);
+        braceSubDefNode.setExpressionType(actual);
 
         return braceSubDefNode;
     }
@@ -75,7 +70,7 @@ boolean isDefOptimized() {
 
     @Override
     void updateActual(Class<?> actual) {
-        this.output.actual = actual;
+        this.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
index f810f1a634b6e..5cface91e59b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
@@ -51,27 +51,23 @@ final class PSubDefCall extends AExpression {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         parameterTypes.add(Object.class);
         int totalCaptures = 0;
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            Input expressionInput = new Input();
-            expressionInput.internal = true;
-            Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
 
-            if (expressionOutput.actual == void.class) {
+            if (expression.actual == void.class) {
                 throw createError(new IllegalArgumentException("Argument(s) cannot be of [void] type when calling method [" + name + "]."));
             }
 
-            expression.input.expected = expressionOutput.actual;
+            expression.expected = expression.actual;
             expression.cast();
-            parameterTypes.add(expressionOutput.actual);
+            parameterTypes.add(expression.actual);
 
             if (expression instanceof ILambda) {
                 ILambda lambda = (ILambda) expression;
@@ -85,9 +81,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
-        output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
-
-        return output;
+        actual = expected == null || expected == ZonedDateTime.class || explicit ? def.class : expected;
     }
 
     @Override
@@ -99,7 +93,7 @@ CallSubDefNode write(ClassNode classNode) {
         }
 
         callSubDefNode.setLocation(location);
-        callSubDefNode.setExpressionType(output.actual);
+        callSubDefNode.setExpressionType(actual);
         callSubDefNode.setName(name);
         callSubDefNode.setRecipe(recipe.toString());
         callSubDefNode.getPointers().addAll(pointers);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
index 75bfa5c5755cc..317fbabd99b87 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
@@ -43,14 +43,9 @@ final class PSubDefField extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
-        output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
-
-        return output;
+        actual = expected == null || expected == ZonedDateTime.class || explicit ? def.class : expected;
     }
 
     @Override
@@ -58,7 +53,7 @@ DotSubDefNode write(ClassNode classNode) {
         DotSubDefNode dotSubDefNode = new DotSubDefNode();
 
         dotSubDefNode.setLocation(location);
-        dotSubDefNode.setExpressionType(output.actual);
+        dotSubDefNode.setExpressionType(actual);
         dotSubDefNode.setValue(value);
 
         return dotSubDefNode;
@@ -71,7 +66,7 @@ boolean isDefOptimized() {
 
     @Override
     void updateActual(Class<?> actual) {
-        this.output.actual = actual;
+        this.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index c9a438571bc36..400dbaa167c38 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -44,18 +44,13 @@ final class PSubField extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
-         if (input.write && Modifier.isFinal(field.javaField.getModifiers())) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+         if (write && Modifier.isFinal(field.javaField.getModifiers())) {
              throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.javaField.getName() + "] " +
                      "for type [" + PainlessLookupUtility.typeToCanonicalTypeName(field.javaField.getDeclaringClass()) + "]."));
          }
 
-         output.actual = field.typeParameter;
-
-         return output;
+        actual = field.typeParameter;
     }
 
     @Override
@@ -63,7 +58,7 @@ DotSubNode write(ClassNode classNode) {
         DotSubNode dotSubNode = new DotSubNode();
 
         dotSubNode.setLocation(location);
-        dotSubNode.setExpressionType(output.actual);
+        dotSubNode.setExpressionType(actual);
         dotSubNode.setField(field);
 
         return dotSubNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 4dd3b46ceaa10..d7ef83e800e02 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -48,10 +48,7 @@ final class PSubListShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
         getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
@@ -71,18 +68,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
-        if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
-            Input indexInput = new Input();
-            indexInput.expected = int.class;
-            index.analyze(scriptRoot, scope, indexInput);
+        if ((read || write) && (!read || getter != null) && (!write || setter != null)) {
+            index.expected = int.class;
+            index.analyze(scriptRoot, scope);
             index.cast();
 
-            output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
+            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + canonicalClassName + "]."));
         }
-
-        return output;
     }
 
     @Override
@@ -92,7 +86,7 @@ ListSubShortcutNode write(ClassNode classNode) {
         listSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
 
         listSubShortcutNode.setLocation(location);
-        listSubShortcutNode.setExpressionType(output.actual);
+        listSubShortcutNode.setExpressionType(actual);
         listSubShortcutNode.setGetter(getter);
         listSubShortcutNode.setSetter(setter);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 707609e5a4947..a1468746fc038 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -48,10 +48,7 @@ final class PSubMapShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
         getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
@@ -70,18 +67,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
-        if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
-            Input indexInput = new Input();
-            indexInput.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
-            index.analyze(scriptRoot, scope, indexInput);
+        if ((read || write) && (!read || getter != null) && (!write || setter != null)) {
+            index.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
+            index.analyze(scriptRoot, scope);
             index.cast();
 
-            output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
+            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + canonicalClassName + "]."));
         }
-
-        return output;
     }
 
     @Override
@@ -91,7 +85,7 @@ MapSubShortcutNode write(ClassNode classNode) {
         mapSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
 
         mapSubShortcutNode.setLocation(location);
-        mapSubShortcutNode.setExpressionType(output.actual);
+        mapSubShortcutNode.setExpressionType(actual);
         mapSubShortcutNode.setGetter(getter);
         mapSubShortcutNode.setSetter(setter);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
index 83d5fded5649d..2f5139457e1f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
@@ -42,17 +42,12 @@ public PSubNullSafeCallInvoke(Location location, AExpression guarded) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        Output guardedOutput = guarded.analyze(scriptRoot, scope, new Input());
-        output.actual = guardedOutput.actual;
-        if (output.actual.isPrimitive()) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        guarded.analyze(scriptRoot, scope);
+        actual = guarded.actual;
+        if (actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
-
-        return output;
     }
 
     @Override
@@ -62,7 +57,7 @@ NullSafeSubNode write(ClassNode classNode) {
         nullSafeSubNode.setChildNode(guarded.write(classNode));
 
         nullSafeSubNode.setLocation(location);
-        nullSafeSubNode.setExpressionType(output.actual);
+        nullSafeSubNode.setExpressionType(actual);
 
         return nullSafeSubNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
index 536c8b15e83c6..5f438644dff70 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
@@ -37,22 +37,16 @@ public PSubNullSafeField(Location location, AStoreable guarded) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.write) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (write) {
             throw createError(new IllegalArgumentException("Can't write to null safe reference"));
         }
-        Input guardedInput = new Input();
-        guardedInput.read = input.read;
-        Output guardedOutput = guarded.analyze(scriptRoot, scope, guardedInput);
-        output.actual = guardedOutput.actual;
-        if (output.actual.isPrimitive()) {
+        guarded.read = read;
+        guarded.analyze(scriptRoot, scope);
+        actual = guarded.actual;
+        if (actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
-
-        return output;
     }
 
     @Override
@@ -72,7 +66,7 @@ NullSafeSubNode write(ClassNode classNode) {
         nullSafeSubNode.setChildNode(guarded.write(classNode));
 
         nullSafeSubNode.setLocation(location);
-        nullSafeSubNode.setExpressionType(output.actual);
+        nullSafeSubNode.setExpressionType(actual);
 
         return nullSafeSubNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index e88fbeaac0765..187df63e303fe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -46,10 +46,7 @@ final class PSubShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         if (getter != null && (getter.returnType == void.class || !getter.typeParameters.isEmpty())) {
             throw createError(new IllegalArgumentException(
                 "Illegal get shortcut on field [" + value + "] for type [" + type + "]."));
@@ -64,13 +61,11 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
-        if ((getter != null || setter != null) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
-            output.actual = setter != null ? setter.typeParameters.get(0) : getter.returnType;
+        if ((getter != null || setter != null) && (!read || getter != null) && (!write || setter != null)) {
+            actual = setter != null ? setter.typeParameters.get(0) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal shortcut on field [" + value + "] for type [" + type + "]."));
         }
-
-        return output;
     }
 
     @Override
@@ -78,7 +73,7 @@ DotSubShortcutNode write(ClassNode classNode) {
         DotSubShortcutNode dotSubShortcutNode = new DotSubShortcutNode();
 
         dotSubShortcutNode.setLocation(location);
-        dotSubShortcutNode.setExpressionType(output.actual);
+        dotSubShortcutNode.setExpressionType(actual);
         dotSubShortcutNode.setGetter(getter);
         dotSubShortcutNode.setSetter(setter);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 8e6db345e0b30..20ee3ec572d57 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -44,10 +44,7 @@ public SBlock(Location location, List<AStatement> statements) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("A block must contain at least one statement."));
         }
@@ -57,25 +54,22 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         for (AStatement statement : statements) {
             // Note that we do not need to check after the last statement because
             // there is no statement that can be unreachable after the last.
-            if (output.allEscape) {
+            if (allEscape) {
                 throw createError(new IllegalArgumentException("Unreachable statement."));
             }
 
-            Input statementInput = new Input();
-            statementInput.inLoop = input.inLoop;
-            statementInput.lastSource = input.lastSource && statement == last;
-            statementInput.lastLoop = (input.beginLoop || input.lastLoop) && statement == last;
-            Output statementOutput = statement.analyze(scriptRoot, scope, statementInput);
-
-            output.methodEscape = statementOutput.methodEscape;
-            output.loopEscape = statementOutput.loopEscape;
-            output.allEscape = statementOutput.allEscape;
-            output.anyContinue |= statementOutput.anyContinue;
-            output.anyBreak |= statementOutput.anyBreak;
-            output.statementCount += statementOutput.statementCount;
+            statement.inLoop = inLoop;
+            statement.lastSource = lastSource && statement == last;
+            statement.lastLoop = (beginLoop || lastLoop) && statement == last;
+            statement.analyze(scriptRoot, scope);
+
+            methodEscape = statement.methodEscape;
+            loopEscape = statement.loopEscape;
+            allEscape = statement.allEscape;
+            anyContinue |= statement.anyContinue;
+            anyBreak |= statement.anyBreak;
+            statementCount += statement.statementCount;
         }
-
-        return output;
     }
 
     @Override
@@ -87,8 +81,8 @@ BlockNode write(ClassNode classNode) {
         }
 
         blockNode.setLocation(location);
-        blockNode.setAllEscape(output.allEscape);
-        blockNode.setStatementCount(output.statementCount);
+        blockNode.setAllEscape(allEscape);
+        blockNode.setStatementCount(statementCount);
 
         return blockNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 4b036153319ea..8c2d2d3c03403 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -35,20 +35,15 @@ public SBreak(Location location) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.inLoop == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!inLoop) {
             throw createError(new IllegalArgumentException("Break statement outside of a loop."));
         }
 
-        output.loopEscape = true;
-        output.allEscape = true;
-        output.anyBreak = true;
-        output.statementCount = 1;
-
-        return output;
+        loopEscape = true;
+        allEscape = true;
+        anyBreak = true;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index f00c0b18a70fc..ed960776a19f2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -46,11 +46,8 @@ public SCatch(Location location, DType baseException, SDeclaration declaration,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        declaration.analyze(scriptRoot, scope, new Input());
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        declaration.analyze(scriptRoot, scope);
 
         Class<?> baseType = baseException.resolveType(scriptRoot.getPainlessLookup()).getType();
         Class<?> type = scope.getVariable(location, declaration.name).getType();
@@ -62,21 +59,18 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (block != null) {
-            Input blockInput = new Input();
-            blockInput.lastSource = input.lastSource;
-            blockInput.inLoop = input.inLoop;
-            blockInput.lastLoop = input.lastLoop;
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
-
-            output.methodEscape = blockOutput.methodEscape;
-            output.loopEscape = blockOutput.loopEscape;
-            output.allEscape = blockOutput.allEscape;
-            output.anyContinue = blockOutput.anyContinue;
-            output.anyBreak = blockOutput.anyBreak;
-            output.statementCount = blockOutput.statementCount;
+            block.lastSource = lastSource;
+            block.inLoop = inLoop;
+            block.lastLoop = lastLoop;
+            block.analyze(scriptRoot, scope);
+
+            methodEscape = block.methodEscape;
+            loopEscape = block.loopEscape;
+            allEscape = block.allEscape;
+            anyContinue = block.anyContinue;
+            anyBreak = block.anyBreak;
+            statementCount = block.statementCount;
         }
-
-        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 1f5c752f8ccef..fe19365095777 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -35,23 +35,18 @@ public SContinue(Location location) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        if (input.inLoop == false) {
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        if (!inLoop) {
             throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
         }
 
-        if (input.lastLoop) {
+        if (lastLoop) {
             throw createError(new IllegalArgumentException("Extraneous continue statement."));
         }
 
-        output.allEscape = true;
-        output.anyContinue = true;
-        output.statementCount = 1;
-
-        return output;
+        allEscape = true;
+        anyContinue = true;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 47bbe0d123ecb..fa85f57346908 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -44,17 +44,12 @@ public SDeclBlock(Location location, List<SDeclaration> declarations) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         for (SDeclaration declaration : declarations) {
-            declaration.analyze(scriptRoot, scope, new Input());
+            declaration.analyze(scriptRoot, scope);
         }
 
-        output.statementCount = declarations.size();
-
-        return output;
+        statementCount = declarations.size();
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 2a28b4d029558..e379881bba154 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DeclarationNode;
-import org.elasticsearch.painless.node.AExpression.Input;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -48,23 +47,17 @@ public SDeclaration(Location location, DType type, String name, boolean requires
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         DResolvedType resolvedType = type.resolveType(scriptRoot.getPainlessLookup());
         type = resolvedType;
 
         if (expression != null) {
-            AExpression.Input expressionInput = new AExpression.Input();
-            expressionInput.expected = resolvedType.getType();
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = resolvedType.getType();
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
 
         scope.defineVariable(location, resolvedType.getType(), name, false);
-
-        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 8d31f325a049f..d95b51d8d1a0a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -45,28 +45,23 @@ public SDo(Location location, SBlock block, AExpression condition) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         scope = scope.newLocalScope();
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
-        Input blockInput = new Input();
-        blockInput.beginLoop = true;
-        blockInput.inLoop = true;
-        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+        block.beginLoop = true;
+        block.inLoop = true;
+        block.analyze(scriptRoot, scope);
 
-        if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
+        if (block.loopEscape && !block.anyContinue) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
-        AExpression.Input conditionInput = new AExpression.Input();
-        conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
+        condition.expected = boolean.class;
+        condition.analyze(scriptRoot, scope);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -76,15 +71,13 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 throw createError(new IllegalArgumentException("Extraneous do while loop."));
             }
 
-            if (blockOutput.anyBreak == false) {
-                output.methodEscape = true;
-                output.allEscape = true;
+            if (!block.anyBreak) {
+                methodEscape = true;
+                allEscape = true;
             }
         }
 
-        output.statementCount = 1;
-
-        return output;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index b5c439233b64d..a64f6451e4dd2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -53,12 +53,9 @@ public SEach(Location location, String type, String name, AExpression expression
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, new AExpression.Input());
-        expression.input.expected = expressionOutput.actual;
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        expression.analyze(scriptRoot, scope);
+        expression.expected = expression.actual;
         expression.cast();
 
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
@@ -70,34 +67,31 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         scope = scope.newLocalScope();
         Variable variable = scope.defineVariable(location, clazz, name, true);
 
-        if (expressionOutput.actual.isArray()) {
+        if (expression.actual.isArray()) {
             sub = new SSubEachArray(location, variable, expression, block);
-        } else if (expressionOutput.actual == def.class || Iterable.class.isAssignableFrom(expressionOutput.actual)) {
+        } else if (expression.actual == def.class || Iterable.class.isAssignableFrom(expression.actual)) {
             sub = new SSubEachIterable(location, variable, expression, block);
         } else {
             throw createError(new IllegalArgumentException("Illegal for each type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expressionOutput.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expression.actual) + "]."));
         }
 
-        sub.analyze(scriptRoot, scope, input);
+        sub.analyze(scriptRoot, scope);
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous for each loop."));
         }
 
-        Input blockInput = new Input();
-        blockInput.beginLoop = true;
-        blockInput.inLoop = true;
-        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
-        blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
+        block.beginLoop = true;
+        block.inLoop = true;
+        block.analyze(scriptRoot, scope);
+        block.statementCount = Math.max(1, block.statementCount);
 
-        if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
+        if (block.loopEscape && !block.anyContinue) {
             throw createError(new IllegalArgumentException("Extraneous for loop."));
         }
 
-        output.statementCount = 1;
-
-        return output;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index f04cefa622acd..2271b63ee2897 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -44,40 +44,34 @@ public SExpression(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         Class<?> rtnType = scope.getReturnType();
         boolean isVoid = rtnType == void.class;
 
-        AExpression.Input expressionInput = new AExpression.Input();
-        expressionInput.read = input.lastSource && !isVoid;
-        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
+        expression.read = lastSource && !isVoid;
+        expression.analyze(scriptRoot, scope);
 
-        if ((input.lastSource == false || isVoid) && expressionOutput.statement == false) {
+        if ((lastSource == false || isVoid) && expression.statement == false) {
             throw createError(new IllegalArgumentException("Not a statement."));
         }
 
-        boolean rtn = input.lastSource && isVoid == false && expressionOutput.actual != void.class;
+        boolean rtn = lastSource && !isVoid && expression.actual != void.class;
 
-        expression.input.expected = rtn ? rtnType : expressionOutput.actual;
-        expression.input.internal = rtn;
+        expression.expected = rtn ? rtnType : expression.actual;
+        expression.internal = rtn;
         expression.cast();
 
-        output = new Output();
-        output.methodEscape = rtn;
-        output.loopEscape = rtn;
-        output.allEscape = rtn;
-        output.statementCount = 1;
-
-        return output;
+        methodEscape = rtn;
+        loopEscape = rtn;
+        allEscape = rtn;
+        statementCount = 1;
     }
 
     @Override
     StatementNode write(ClassNode classNode) {
         ExpressionNode expressionNode = expression.cast(expression.write(classNode));
 
-        if (output.methodEscape) {
+        if (methodEscape) {
             ReturnNode returnNode = new ReturnNode();
 
             returnNode.setExpressionNode(expressionNode);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 241c50b8fe2b9..a3f71376e061c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -52,26 +52,23 @@ public SFor(Location location, ANode initializer, AExpression condition, AExpres
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         scope = scope.newLocalScope();
 
         if (initializer != null) {
             if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(scriptRoot, scope, new Input());
+                ((SDeclBlock)initializer).analyze(scriptRoot, scope);
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
-                AExpression.Input initializerInput = new AExpression.Input();
-                initializerInput.read = false;
-                AExpression.Output initializerOutput = initializer.analyze(scriptRoot, scope, initializerInput);
+                initializer.read = false;
+                initializer.analyze(scriptRoot, scope);
 
-                if (initializerOutput.statement == false) {
+                if (!initializer.statement) {
                     throw createError(new IllegalArgumentException("Not a statement."));
                 }
 
-                initializer.input.expected = initializerOutput.actual;
+                initializer.expected = initializer.actual;
                 initializer.cast();
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
@@ -79,9 +76,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (condition != null) {
-            AExpression.Input conditionInput = new AExpression.Input();
-            conditionInput.expected = boolean.class;
-            condition.analyze(scriptRoot, scope, conditionInput);
+            condition.expected = boolean.class;
+            condition.analyze(scriptRoot, scope);
             condition.cast();
 
             if (condition instanceof EBoolean) {
@@ -100,42 +96,36 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (afterthought != null) {
-            AExpression.Input afterthoughtInput = new AExpression.Input();
-            afterthoughtInput.read = false;
-            AExpression.Output afterthoughtOutput = afterthought.analyze(scriptRoot, scope, afterthoughtInput);
+            afterthought.read = false;
+            afterthought.analyze(scriptRoot, scope);
 
-            if (afterthoughtOutput.statement == false) {
+            if (!afterthought.statement) {
                 throw createError(new IllegalArgumentException("Not a statement."));
             }
 
-            afterthought.input.expected = afterthoughtOutput.actual;
+            afterthought.expected = afterthought.actual;
             afterthought.cast();
         }
 
-        output = new Output();
-
         if (block != null) {
-            Input blockInput = new Input();
-            blockInput.beginLoop = true;
-            blockInput.inLoop = true;
+            block.beginLoop = true;
+            block.inLoop = true;
 
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            block.analyze(scriptRoot, scope);
 
-            if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
+            if (block.loopEscape && !block.anyContinue) {
                 throw createError(new IllegalArgumentException("Extraneous for loop."));
             }
 
-            if (continuous && blockOutput.anyBreak == false) {
-                output.methodEscape = true;
-                output.allEscape = true;
+            if (continuous && !block.anyBreak) {
+                methodEscape = true;
+                allEscape = true;
             }
 
-            blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
+            block.statementCount = Math.max(1, block.statementCount);
         }
 
-        output.statementCount = 1;
-
-        return output;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 96bf040099a17..2ae9ef8d5d78f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -30,8 +30,6 @@
 import org.elasticsearch.painless.ir.ReturnNode;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.node.AStatement.Input;
-import org.elasticsearch.painless.node.AStatement.Output;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.lang.invoke.MethodType;
@@ -136,10 +134,9 @@ void analyze(ScriptRoot scriptRoot) {
             throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "]."));
         }
 
-        Input blockInput = new Input();
-        blockInput.lastSource = true;
-        Output blockOutput = block.analyze(scriptRoot, functionScope.newLocalScope(), blockInput);
-        methodEscape = blockOutput.methodEscape;
+        block.lastSource = true;
+        block.analyze(scriptRoot, functionScope.newLocalScope());
+        methodEscape = block.methodEscape;
 
         if (methodEscape == false && isAutoReturnEnabled == false && returnType != void.class) {
             throw createError(new IllegalArgumentException("not all paths provide a return value " +
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index a7d738670ec6f..3b95ec181dc68 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -43,13 +43,9 @@ public SIf(Location location, AExpression condition, SBlock ifblock) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        AExpression.Input conditionInput = new AExpression.Input();
-        conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        condition.expected = boolean.class;
+        condition.analyze(scriptRoot, scope);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -60,18 +56,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
-        Input ifblockInput = new Input();
-        ifblockInput.lastSource = input.lastSource;
-        ifblockInput.inLoop = input.inLoop;
-        ifblockInput.lastLoop = input.lastLoop;
-
-        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
+        ifblock.lastSource = lastSource;
+        ifblock.inLoop = inLoop;
+        ifblock.lastLoop = lastLoop;
 
-        output.anyContinue = ifblockOutput.anyContinue;
-        output.anyBreak = ifblockOutput.anyBreak;
-        output.statementCount = ifblockOutput.statementCount;
+        ifblock.analyze(scriptRoot, scope.newLocalScope());
 
-        return output;
+        anyContinue = ifblock.anyContinue;
+        anyBreak = ifblock.anyBreak;
+        statementCount = ifblock.statementCount;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index f433db860ccfb..b3726cee6c114 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -48,13 +48,9 @@ public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        AExpression.Input conditionInput = new AExpression.Input();
-        conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        condition.expected = boolean.class;
+        condition.analyze(scriptRoot, scope);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -65,36 +61,32 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
-        Input ifblockInput = new Input();
-        ifblockInput.lastSource = input.lastSource;
-        ifblockInput.inLoop = input.inLoop;
-        ifblockInput.lastLoop = input.lastLoop;
+        ifblock.lastSource = lastSource;
+        ifblock.inLoop = inLoop;
+        ifblock.lastLoop = lastLoop;
 
-        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
+        ifblock.analyze(scriptRoot, scope.newLocalScope());
 
-        output.anyContinue = ifblockOutput.anyContinue;
-        output.anyBreak = ifblockOutput.anyBreak;
-        output.statementCount = ifblockOutput.statementCount;
+        anyContinue = ifblock.anyContinue;
+        anyBreak = ifblock.anyBreak;
+        statementCount = ifblock.statementCount;
 
         if (elseblock == null) {
             throw createError(new IllegalArgumentException("Extraneous else statement."));
         }
 
-        Input elseblockInput = new Input();
-        elseblockInput.lastSource = input.lastSource;
-        elseblockInput.inLoop = input.inLoop;
-        elseblockInput.lastLoop = input.lastLoop;
-
-        Output elseblockOutput = elseblock.analyze(scriptRoot, scope.newLocalScope(), elseblockInput);
+        elseblock.lastSource = lastSource;
+        elseblock.inLoop = inLoop;
+        elseblock.lastLoop = lastLoop;
 
-        output.methodEscape = ifblockOutput.methodEscape && elseblockOutput.methodEscape;
-        output.loopEscape = ifblockOutput.loopEscape && elseblockOutput.loopEscape;
-        output.allEscape = ifblockOutput.allEscape && elseblockOutput.allEscape;
-        output.anyContinue |= elseblockOutput.anyContinue;
-        output.anyBreak |= elseblockOutput.anyBreak;
-        output.statementCount = Math.max(ifblockOutput.statementCount, elseblockOutput.statementCount);
+        elseblock.analyze(scriptRoot, scope.newLocalScope());
 
-        return output;
+        methodEscape = ifblock.methodEscape && elseblock.methodEscape;
+        loopEscape = ifblock.loopEscape && elseblock.loopEscape;
+        allEscape = ifblock.allEscape && elseblock.allEscape;
+        anyContinue |= elseblock.anyContinue;
+        anyBreak |= elseblock.anyBreak;
+        statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 23a5d4183edbc..e1d181d78025b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -40,10 +40,7 @@ public SReturn(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         if (expression == null) {
             if (scope.getReturnType() != void.class) {
                 throw location.createError(new ClassCastException("Cannot cast from " +
@@ -51,20 +48,17 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(void.class) + "]."));
             }
         } else {
-            AExpression.Input expressionInput = new AExpression.Input();
-            expressionInput.expected = scope.getReturnType();
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
+            expression.expected = scope.getReturnType();
+            expression.internal = true;
+            expression.analyze(scriptRoot, scope);
             expression.cast();
         }
 
-        output.methodEscape = true;
-        output.loopEscape = true;
-        output.allEscape = true;
-
-        output.statementCount = 1;
+        methodEscape = true;
+        loopEscape = true;
+        allEscape = true;
 
-        return output;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index 5e1846457d515..e7c954828eb07 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -52,18 +52,13 @@ final class SSubEachArray extends AStatement {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
-        array = scope.defineInternalVariable(location, expression.output.actual, "array" + location.getOffset(), true);
+        array = scope.defineInternalVariable(location, expression.actual, "array" + location.getOffset(), true);
         index = scope.defineInternalVariable(location, int.class, "index" + location.getOffset(), true);
-        indexed = expression.output.actual.getComponentType();
+        indexed = expression.actual.getComponentType();
         cast = AnalyzerCaster.getLegalCast(location, indexed, variable.getType(), true, true);
-
-        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 4198452ff8e79..72d29b9f03ddc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -57,28 +57,23 @@ final class SSubEachIterable extends AStatement {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
         iterator = scope.defineInternalVariable(location, Iterator.class, "itr" + location.getOffset(), true);
 
-        if (expression.output.actual == def.class) {
+        if (expression.actual == def.class) {
             method = null;
         } else {
-            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expression.output.actual, false, "iterator", 0);
+            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expression.actual, false, "iterator", 0);
 
             if (method == null) {
                     throw createError(new IllegalArgumentException(
-                            "method [" + typeToCanonicalTypeName(expression.output.actual) + ", iterator/0] not found"));
+                            "method [" + typeToCanonicalTypeName(expression.actual) + ", iterator/0] not found"));
             }
         }
 
         cast = AnalyzerCaster.getLegalCast(location, def.class, variable.getType(), true, true);
-
-        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 86b40fafa378b..e3132f024f52b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -41,21 +41,15 @@ public SThrow(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        AExpression.Input expressionInput = new AExpression.Input();
-        expressionInput.expected = Exception.class;
-        expression.analyze(scriptRoot, scope, expressionInput);
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
+        expression.expected = Exception.class;
+        expression.analyze(scriptRoot, scope);
         expression.cast();
 
-        output.methodEscape = true;
-        output.loopEscape = true;
-        output.allEscape = true;
-        output.statementCount = 1;
-
-        return output;
+        methodEscape = true;
+        loopEscape = true;
+        allEscape = true;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 7798fd13b13a2..6c73b625a64ac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -46,49 +46,42 @@ public STry(Location location, SBlock block, List<SCatch> catches) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous try statement."));
         }
 
-        Input blockInput = new Input();
-        blockInput.lastSource = input.lastSource;
-        blockInput.inLoop = input.inLoop;
-        blockInput.lastLoop = input.lastLoop;
+        block.lastSource = lastSource;
+        block.inLoop = inLoop;
+        block.lastLoop = lastLoop;
 
-        Output blockOutput = block.analyze(scriptRoot, scope.newLocalScope(), blockInput);
+        block.analyze(scriptRoot, scope.newLocalScope());
 
-        output.methodEscape = blockOutput.methodEscape;
-        output.loopEscape = blockOutput.loopEscape;
-        output.allEscape = blockOutput.allEscape;
-        output.anyContinue = blockOutput.anyContinue;
-        output.anyBreak = blockOutput.anyBreak;
+        methodEscape = block.methodEscape;
+        loopEscape = block.loopEscape;
+        allEscape = block.allEscape;
+        anyContinue = block.anyContinue;
+        anyBreak = block.anyBreak;
 
         int statementCount = 0;
 
         for (SCatch catc : catches) {
-            Input catchInput = new Input();
-            catchInput.lastSource = input.lastSource;
-            catchInput.inLoop = input.inLoop;
-            catchInput.lastLoop = input.lastLoop;
+            catc.lastSource = lastSource;
+            catc.inLoop = inLoop;
+            catc.lastLoop = lastLoop;
 
-            Output catchOutput = catc.analyze(scriptRoot, scope.newLocalScope(), catchInput);
+            catc.analyze(scriptRoot, scope.newLocalScope());
 
-            output.methodEscape &= catchOutput.methodEscape;
-            output.loopEscape &= catchOutput.loopEscape;
-            output.allEscape &= catchOutput.allEscape;
-            output.anyContinue |= catchOutput.anyContinue;
-            output.anyBreak |= catchOutput.anyBreak;
+            methodEscape &= catc.methodEscape;
+            loopEscape &= catc.loopEscape;
+            allEscape &= catc.allEscape;
+            anyContinue |= catc.anyContinue;
+            anyBreak |= catc.anyBreak;
 
-            statementCount = Math.max(statementCount, catchOutput.statementCount);
+            statementCount = Math.max(statementCount, catc.statementCount);
         }
 
-        output.statementCount = blockOutput.statementCount + statementCount;
-
-        return output;
+        this.statementCount = block.statementCount + statementCount;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index eead51786e2d3..8213e5e1bc559 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -45,15 +45,11 @@ public SWhile(Location location, AExpression condition, SBlock block) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    void analyze(ScriptRoot scriptRoot, Scope scope) {
         scope = scope.newLocalScope();
 
-        AExpression.Input conditionInput = new AExpression.Input();
-        conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
+        condition.expected = boolean.class;
+        condition.analyze(scriptRoot, scope);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -69,27 +65,24 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (block != null) {
-            Input blockInput = new Input();
-            blockInput.beginLoop = true;
-            blockInput.inLoop = true;
+            block.beginLoop = true;
+            block.inLoop = true;
 
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            block.analyze(scriptRoot, scope);
 
-            if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
+            if (block.loopEscape && !block.anyContinue) {
                 throw createError(new IllegalArgumentException("Extraneous while loop."));
             }
 
-            if (continuous && blockOutput.anyBreak == false) {
-                output.methodEscape = true;
-                output.allEscape = true;
+            if (continuous && !block.anyBreak) {
+                methodEscape = true;
+                allEscape = true;
             }
 
-            blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
+            block.statementCount = Math.max(1, block.statementCount);
         }
 
-        output.statementCount = 1;
-
-        return output;
+        statementCount = 1;
     }
 
     @Override
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
index 6fb5b44d957aa..adde835deb104 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
@@ -28,6 +28,7 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.BlendedTermQuery;
+import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.IntervalQuery;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
@@ -1121,33 +1122,33 @@ public void testIntervalQueries() {
         assertTermsEqual(result.extractions, new Term("field", "term1"), new Term("field", "term2"),
             new Term("field", "term3"), new Term("field", "term4"));
 
-        source = Intervals.ordered(Intervals.term("term1"), Intervals.wildcard(new BytesRef("a*")));
+        source = Intervals.ordered(Intervals.term("term1"), XIntervals.wildcard(new BytesRef("a*")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertThat(result.verified, is(false));
         assertThat(result.matchAllDocs, is(false));
         assertThat(result.minimumShouldMatch, equalTo(1));
         assertTermsEqual(result.extractions, new Term("field", "term1"));
 
-        source = Intervals.ordered(Intervals.wildcard(new BytesRef("a*")));
+        source = Intervals.ordered(XIntervals.wildcard(new BytesRef("a*")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
-        source = Intervals.or(Intervals.term("b"), Intervals.wildcard(new BytesRef("a*")));
+        source = Intervals.or(Intervals.term("b"), XIntervals.wildcard(new BytesRef("a*")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
-        source = Intervals.ordered(Intervals.term("term1"), Intervals.prefix(new BytesRef("a")));
+        source = Intervals.ordered(Intervals.term("term1"), XIntervals.prefix(new BytesRef("a")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertThat(result.verified, is(false));
         assertThat(result.matchAllDocs, is(false));
         assertThat(result.minimumShouldMatch, equalTo(1));
         assertTermsEqual(result.extractions, new Term("field", "term1"));
 
-        source = Intervals.ordered(Intervals.prefix(new BytesRef("a")));
+        source = Intervals.ordered(XIntervals.prefix(new BytesRef("a")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
-        source = Intervals.or(Intervals.term("b"), Intervals.prefix(new BytesRef("a")));
+        source = Intervals.or(Intervals.term("b"), XIntervals.prefix(new BytesRef("a")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index f639c17a35f5a..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b6f880fa08a44fcb2d50808f9eeb6189a293ce27
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..d90402c25e434
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+0748be5811dfe6725847d2e87890a990c58cc3de
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 5c1e0b2f6fa43..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9c5b8619795f69c225b5ec37b87cb34de0feccd4
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..a5003b5fb51a2
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+f693cd60ad8ca9b7d3082f7b9ee6054b9c819b48
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 656f63c68d194..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-421e13b9fe09523e094ac708204d62d4ea5b6618
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..d82cbaef39b0b
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+72c34e18af81ee1d18e9927fb95690fe056cbd4f
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 951d97032f9fb..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ff4ae9f3f3b0bc497f98c9bc47e943525669fc99
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..2b08ae87a25d9
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+655438348dcad9a98b5affa76caa3d67aa4bee51
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 19846e43636c3..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd6430c037566cd3852b73b2ec31e59de24cfe58
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..9a051c8320ae1
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+b99147dad649fce0b0423e41f90c79e0f2fba2b7
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 147bbb3192d24..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd4ca22b151a98a21e255bc1c54f0fadfee5ca4d
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..92d6378f04c08
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+00ce3e23cf7aba8c1b3e777de92fd31ec1d4d814
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 86cee0ecdd039..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e85f94d2747ddb560af0bc4d15f0cde45cf3ff30
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..427fa3cc39049
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+2b03f79d61517d8e6a8744dbd89e61ad661f6a62
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
new file mode 100644
index 0000000000000..5203969bcf5c0
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
@@ -0,0 +1 @@
+596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
new file mode 100644
index 0000000000000..5203969bcf5c0
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
@@ -0,0 +1 @@
+596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java
index f7a621c4125a9..c473e4e010552 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.SegmentCommitInfo;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.index.mapper.ParsedDocument;
 
 import java.io.IOException;
@@ -90,7 +89,7 @@ public synchronized MergePolicy.OneMerge getNextMerge() {
                         StreamSupport.stream(e.getLastCommittedSegmentInfos().spliterator(), false).collect(Collectors.toList());
                 segmentsReference.set(segments);
                 // trigger a background merge that will be managed by the concurrent merge scheduler
-                e.forceMerge(randomBoolean(), 0, false, false, false, UUIDs.randomBase64UUID());
+                e.forceMerge(randomBoolean(), 0, false, false, false);
                 /*
                  * Merging happens in the background on a merge thread, and the maybeDie handler is invoked on yet another thread; we have
                  * to wait for these events to finish.
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
index f8a31c5ec9214..0e5236f9b1171 100644
--- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
@@ -80,9 +80,9 @@
       cluster.remote_info: {}
 
   - match: { test_remote_cluster.connected: true }
-  - match: { test_remote_cluster.proxy_address: $remote_ip }
-  - gt:    { test_remote_cluster.num_proxy_sockets_connected: 0}
-  - match: { test_remote_cluster.max_proxy_socket_connections: 10}
+  - match: { test_remote_cluster.address: $remote_ip }
+  - gt:    { test_remote_cluster.num_sockets_connected: 0}
+  - match: { test_remote_cluster.max_socket_connections: 10}
   - match: { test_remote_cluster.initial_connect_timeout: "30s" }
   - match: { test_remote_cluster.mode: "proxy" }
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
index 6ee594b099c43..d37273cfd41bf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
@@ -53,18 +53,6 @@
         "type":"boolean",
         "description":"Verbose mode. Display column headers",
         "default":false
-      },
-      "expand_wildcards":{
-        "type":"enum",
-        "options":[
-          "open",
-          "closed",
-          "hidden",
-          "none",
-          "all"
-        ],
-        "default": ["all"],
-        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
index d4a7eb3b051d2..76c749cc7d3bd 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
@@ -107,18 +107,6 @@
         "type":"boolean",
         "description":"If set to true segment stats will include stats for segments that are not currently loaded into memory",
         "default":false
-      },
-      "expand_wildcards":{
-        "type":"enum",
-        "options":[
-          "open",
-          "closed",
-          "hidden",
-          "none",
-          "all"
-        ],
-        "default": "all",
-        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
index 2a21ff3725b6a..7911a8e244218 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
@@ -33,7 +33,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
index 017705082d189..d6f0e9ababc6d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
@@ -97,7 +97,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
index 93a450afff88a..4084d3be608a4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -47,7 +47,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index 9be09b49cc54c..93f6d63bbc2d2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -70,7 +70,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
index d56c3313a0bf0..eed6f8bf082a2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
@@ -47,7 +47,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
index 64c10a520c7c4..b1b1d8214c47d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
@@ -53,7 +53,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
index f26c8e77a06a6..539088199d683 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
@@ -43,7 +43,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
index 53fdf44bb36a1..d77d47f0d739b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
@@ -43,7 +43,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
index 7539f44a81eed..d6bdcef95815f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
@@ -39,7 +39,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
index 66e5ce92cbbe5..a19786a7938f9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
@@ -51,7 +51,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
index c854d0e8fd841..2a09d8bdfa679 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
@@ -39,7 +39,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
index 35138b920466f..1eada8f831db6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
@@ -51,7 +51,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
index 6036b75bb83e4..5eaf62840d841 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
@@ -45,7 +45,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
index 90a1274ecb059..ccacf91d93095 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
@@ -39,7 +39,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
index e238c4fc38afc..800ea8d30d68d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
@@ -69,11 +69,10 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
-        "default": ["all"],
+        "default":"all",
         "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       },
       "local":{
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
index 0e71b6d395777..4c5191f2f320a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
@@ -55,7 +55,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
index 5a27f6d06b83c..17274e8bbd78a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
@@ -41,7 +41,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
index 4a1dea974f750..c19fa3bf5262f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
@@ -73,7 +73,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
index 68cfdf25aeea2..862fdde4ac395 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
@@ -49,7 +49,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
index 1dab468ce4ff4..14050d626df83 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
@@ -43,7 +43,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
index 18a42ec11a45e..c95f5ea3b48f3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
@@ -44,7 +44,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
index 66fe23bab8ba2..75bf3a340c311 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
@@ -53,7 +53,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
index 950e0a62489fa..7eee4eeec5fd0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
@@ -43,7 +43,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
index 83430a9a85600..b99b19799dab9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
@@ -41,7 +41,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
index 7e48e99916171..b397963f99d2f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
@@ -51,7 +51,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
index 0a8960f2f9e89..0940313154a74 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
@@ -137,7 +137,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
index 406fbacda307c..93628fdd106d0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
@@ -45,7 +45,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
index 3becec003a9e6..5ff6f073b77ec 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
@@ -69,7 +69,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
index eadf240192394..c87b850ab148f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
@@ -43,7 +43,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 1e0c232efa055..2411c309c7dff 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -89,7 +89,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
index 74b7055b4c4b0..70ee592520be9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
@@ -55,7 +55,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
index 1d239376b3fe8..f3286cd932c4e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -47,7 +47,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
index 299c4f705d292..591371798d440 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
@@ -70,7 +70,6 @@
         "options":[
           "open",
           "closed",
-          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
deleted file mode 100644
index 6866ff595fefd..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
+++ /dev/null
@@ -1,150 +0,0 @@
----
-"Test cat aliases output with a hidden index with a hidden alias":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-            index:
-              hidden: true
-          aliases:
-            test_alias:
-              is_hidden: true
-
-  - do:
-      cat.aliases: {}
-
-  - match:
-      $body: |
-        /^
-            test_alias          \s+
-            test                \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-        $/
-
-  - do:
-      cat.aliases:
-        name: test_alias
-
-  - match:
-      $body: |
-        /^
-            test_alias          \s+
-            test                \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-        $/
-
-
-  - do:
-      cat.aliases:
-        expand_wildcards: ["open","closed"]
-
-  - match:
-      $body: |
-        /^
-        $/
----
-"Test cat aliases output with a hidden index with a visible alias":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-            index:
-              hidden: true
-          aliases:
-            test_alias: {}
-  - do:
-      cat.aliases: {}
-
-  - match:
-      $body: |
-        /^
-            test_alias          \s+
-            test                \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-        $/
-
-  - do:
-      cat.aliases:
-        name: test_alias
-
-  - match:
-      $body: |
-        /^
-            test_alias          \s+
-            test                \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-        $/
-
----
-"Test cat aliases output with a visible index with a hidden alias":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-          aliases:
-            test_alias:
-              is_hidden: true
-  - do:
-      cat.aliases: {}
-
-  - match:
-      $body: |
-        /^
-            test_alias          \s+
-            test                \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-        $/
-
-  - do:
-      cat.aliases:
-        name: test_alias
-
-  - match:
-      $body: |
-        /^
-            test_alias          \s+
-            test                \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-            -                   \s+
-        $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
deleted file mode 100644
index 3a4fe28c85996..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
+++ /dev/null
@@ -1,243 +0,0 @@
----
-"Test cat indices output for hidden index":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-  - do:
-      indices.create:
-        index: index1
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-            index:
-              hidden: true
-  - do:
-      cat.indices: {}
-  - match:
-      $body: |
-        /^$/
-
-  - do:
-      cat.indices:
-        expand_wildcards: ["all"]
-  - match:
-      $body: |
-        /^(green  \s+
-           open   \s+
-           index1 \s+
-           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
-           1      \s+
-           0      \s+
-           0      \s+
-           0      \s+
-           (\d+|\d+[.]\d+)(kb|b) \s+
-           (\d+|\d+[.]\d+)(kb|b) \s*
-         )
-         $/
-
----
-"Test cat indices output for dot-hidden index and dot-prefixed pattern":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-  - do:
-      indices.create:
-        index: .index1
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-            index:
-              hidden: true
-  - do:
-      cat.indices: {}
-  - match:
-      $body: |
-        /^$/
-
-  - do:
-      cat.indices:
-        index: ".*"
-  - match:
-      $body: |
-        /^(green  \s+
-           open   \s+
-           \.index1 \s+
-           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
-           1      \s+
-           0      \s+
-           0      \s+
-           0      \s+
-           (\d+|\d+[.]\d+)(kb|b) \s+
-           (\d+|\d+[.]\d+)(kb|b) \s*
-         )
-         $/
-
----
-"Test cat indices output with a hidden index with a visible alias":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-
-  - do:
-      indices.create:
-        index: index1
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-            index:
-              hidden: true
-          aliases:
-            alias1: {}
-  - do:
-      cat.indices:
-        index: "i*"
-        # Can't use a bare wildcard here because Security replaces wildcards
-        # it with all matching authorized indices/aliases, including the visible
-        # alias
-  - match:
-      $body: |
-        /^$/
-
-  - do:
-      cat.indices:
-        expand_wildcards: ["open", "hidden"]
-  - match:
-      $body: |
-        /^(green  \s+
-           open   \s+
-           index1 \s+
-           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
-           1      \s+
-           0      \s+
-           0      \s+
-           0      \s+
-           (\d+|\d+[.]\d+)(kb|b) \s+
-           (\d+|\d+[.]\d+)(kb|b) \s*
-         )
-         $/
-
-  - do:
-      cat.indices:
-        index: alias1
-  - match:
-      $body: |
-        /^(green  \s+
-           open   \s+
-           index1 \s+
-           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
-           1      \s+
-           0      \s+
-           0      \s+
-           0      \s+
-           (\d+|\d+[.]\d+)(kb|b) \s+
-           (\d+|\d+[.]\d+)(kb|b) \s*
-         )
-         $/
----
-"Test cat indices output with a hidden index with a hidden alias":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-
-  - do:
-      indices.create:
-        index: index1
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-            index:
-              hidden: true
-          aliases:
-            alias1:
-              is_hidden: true
-  - do:
-      cat.indices: {}
-
-  - match:
-      $body: |
-        /^$/
-
-  - do:
-      cat.indices:
-        expand_wildcards: ["all"]
-  - match:
-      $body: |
-        /^(green  \s+
-           open   \s+
-           index1 \s+
-           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
-           1      \s+
-           0      \s+
-           0      \s+
-           0      \s+
-           (\d+|\d+[.]\d+)(kb|b) \s+
-           (\d+|\d+[.]\d+)(kb|b) \s*
-         )
-         $/
-
-  - do:
-      cat.indices:
-        index: alias1
-  - match:
-      $body: |
-        /^(green  \s+
-           open   \s+
-           index1 \s+
-           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
-           1      \s+
-           0      \s+
-           0      \s+
-           0      \s+
-           (\d+|\d+[.]\d+)(kb|b) \s+
-           (\d+|\d+[.]\d+)(kb|b) \s*
-         )
-         $/
----
-"Test cat indices output with a hidden index, dot-hidden alias and dot pattern":
-  - skip:
-      version: "- 7.99.99"
-      reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
-
-  - do:
-      indices.create:
-        index: index1
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-            index:
-              hidden: true
-          aliases:
-            .alias1:
-              is_hidden: true
-  - do:
-      cat.indices: {}
-  - match:
-      $body: |
-        /^$/
-  - do:
-      cat.indices:
-        index: ".*"
-  - match:
-      $body: |
-        /^(green  \s+
-           open   \s+
-           index1 \s+
-           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
-           1      \s+
-           0      \s+
-           0      \s+
-           0      \s+
-           (\d+|\d+[.]\d+)(kb|b) \s+
-           (\d+|\d+[.]\d+)(kb|b) \s*
-         )
-         $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
index be3d6c60a3fee..49afbb8afc714 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
@@ -8,8 +8,6 @@ setup:
                 properties:
                   date:
                     type: date
-                  date_nanos:
-                    type: date_nanos
                   keyword:
                     type: keyword
                   long:
@@ -852,138 +850,3 @@ setup:
   - length: { aggregations.test.buckets: 1 }
   - match: { aggregations.test.buckets.0.key.date: "2017-10-21T00:00:00.000-02:00" }
   - match: { aggregations.test.buckets.0.doc_count: 2 }
-
----
-"date_histogram on date_nanos":
-  - skip:
-      version: " - 7.6.99"
-      reason: Fixed in 7.7.0
-  - do:
-      index:
-        index:   test
-        id:      7
-        body:    { "date_nanos": "2017-11-21T01:00:00" }
-        refresh: true
-  - do:
-      index:
-        index:   test
-        id:      8
-        body:    { "date_nanos": "2017-11-22T01:00:00" }
-        refresh: true
-  - do:
-      index:
-        index:   test
-        id:      9
-        body:    { "date_nanos": "2017-11-22T02:00:00" }
-        refresh: true
-  - do:
-      search:
-        index: test
-        body:
-          aggregations:
-            test:
-              composite:
-                sources:
-                  - date:
-                      date_histogram:
-                        field: date_nanos
-                        calendar_interval: 1d
-                        format: iso8601 # Format makes the comparisons a little more obvious
-              aggregations:
-                avg:
-                  avg:
-                    field: date_nanos
-
-  - match: { hits.total.value: 9 }
-  - match: { hits.total.relation: eq }
-  - length: { aggregations.test.buckets: 2 }
-  - match: { aggregations.test.buckets.0.key.date: "2017-11-21T00:00:00.000Z" }
-  - match: { aggregations.test.buckets.0.doc_count: 1 }
-  - match: { aggregations.test.buckets.0.avg.value_as_string: "2017-11-21T01:00:00.000Z" }
-  - match: { aggregations.test.buckets.1.key.date: "2017-11-22T00:00:00.000Z" }
-  - match: { aggregations.test.buckets.1.doc_count: 2 }
-  - match: { aggregations.test.buckets.1.avg.value_as_string: "2017-11-22T01:30:00.000Z" }
-
----
-"Terms source from sorted":
-  - do:
-        indices.create:
-          index: sorted_test
-          body:
-              settings:
-                sort.field: keyword
-              mappings:
-                properties:
-                  keyword:
-                    type: keyword
-                  long:
-                    type: long
-
-
-  - do:
-      index:
-        index:   sorted_test
-        id:      2
-        refresh: true
-        body:  { "keyword": "foo", "long": 1 }
-
-  - do:
-      search:
-        index: sorted_test
-        body:
-          aggregations:
-            test:
-              composite:
-                sources:
-                  - keyword:
-                      terms:
-                        field: keyword
-
-  - match: {hits.total.value: 1}
-  - length: { aggregations.test.buckets: 1 }
-  - match: { aggregations.test.buckets.0.key.keyword: "foo" }
-  - match: { aggregations.test.buckets.0.doc_count: 1 }
-
----
-"Terms source from part of sorted":
-  - skip:
-      version: " - 7.6.99"
-      reason:  fixed in 7.7.0.
-
-  - do:
-        indices.create:
-          index: sorted_test
-          body:
-              settings:
-                sort.field: [keyword, long]
-              mappings:
-                properties:
-                  keyword:
-                    type: keyword
-                  long:
-                    type: long
-
-
-  - do:
-      index:
-        index:   sorted_test
-        id:      2
-        refresh: true
-        body:  { "keyword": "foo", "long": 1 }
-
-  - do:
-      search:
-        index: sorted_test
-        body:
-          aggregations:
-            test:
-              composite:
-                sources:
-                  - keyword:
-                      terms:
-                        field: keyword
-
-  - match: {hits.total.value: 1}
-  - length: { aggregations.test.buckets: 1 }
-  - match: { aggregations.test.buckets.0.key.keyword: "foo" }
-  - match: { aggregations.test.buckets.0.doc_count: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml
deleted file mode 100644
index bd7bb925a7adb..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml
+++ /dev/null
@@ -1,98 +0,0 @@
-setup:
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            number_of_replicas: 0
-          mappings:
-            properties:
-              field1:
-                type: long
-              field2:
-                type: long
-              field3:
-                type: long
-  - do:
-      bulk:
-        refresh: true
-        body:
-          - index:
-              _index: test
-          - field1: 100
-          - index:
-              _index: test
-          - field1: 200
-          - index:
-              _index: test
-          - field1: 300
-            field2: 300
-
----
-"match all":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          aggs:
-            missing_agg:
-              missing:
-                field: field3
-
-  - match: { hits.total: 3 }
-  - length: { hits.hits: 3 }
-
-  - match: { aggregations.missing_agg.doc_count: 3 }
-
----
-"match some":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          aggs:
-            missing_agg:
-              missing:
-                field: field2
-
-  - match: { hits.total: 3 }
-  - length: { hits.hits: 3 }
-
-  - match: { aggregations.missing_agg.doc_count: 2 }
-
----
-"match none":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          aggs:
-            missing_agg:
-              missing:
-                field: field1
-
-  - match: { hits.total: 3 }
-  - length: { hits.hits: 3 }
-
-  - match: { aggregations.missing_agg.doc_count: 0 }
-
----
-"missing param":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          aggs:
-            missing_agg:
-              missing:
-                field: field3
-                missing: 1
-
-  - match: { hits.total: 3 }
-  - length: { hits.hits: 3 }
-
-  - match: { aggregations.missing_agg.doc_count: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
deleted file mode 100644
index 16442d2b3d823..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
+++ /dev/null
@@ -1,64 +0,0 @@
----
-setup:
-  - do:
-      indices.create:
-          index: test-index
-          body:
-            mappings:
-              "properties":
-                "k1":
-                  "type": "keyword"
-                "k2":
-                  "type": "keyword"
-                  "ignore_above": 3
-  - do:
-      bulk:
-        index: test-index
-        refresh: true
-        body:
-          - '{"index": {"_id": "1"}}'
-          - '{"k1": "123", "k2" : "123"}'
-          - '{"index": {"_id": "2"}}'
-          - '{"k1": "1234", "k2" : "1234"}'
-
----
-"Plain Highligher should skip highlighting ignored keyword values":
-  - skip:
-      version: " - 7.9.99"
-      reason: "skip highlighting of ignored values was introduced in 7.7"
-  - do:
-      search:
-        index: test-index
-        body:
-          query:
-            prefix:
-              k1: "12"
-          highlight:
-            require_field_match: false
-            fields:
-              k2:
-                type: plain
-
-  - match: {hits.hits.0.highlight.k2.0: "<em>123</em>"}
-  - is_false: hits.hits.1.highlight # no highlight for a value that was ignored
-
----
-"Unified Highligher should skip highlighting ignored keyword values":
-  - skip:
-      version: " - 7.9.99"
-      reason: "skip highlighting of ignored values was introduced in 7.7"
-  - do:
-      search:
-        index: test-index
-        body:
-          query:
-            prefix:
-              k1: "12"
-          highlight:
-            require_field_match: false
-            fields:
-              k2:
-                type: unified
-
-  - match: {hits.hits.0.highlight.k2.0: "<em>123</em>"}
-  - is_false: hits.hits.1.highlight # no highlight for a value that was ignored
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
index 7135c8642736c..e0b6827aa7995 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
@@ -55,7 +55,94 @@ teardown:
       cluster.get_settings:
         flat_settings: true
 
-  - is_false: search.allow_expensive_queries
+  - match: {search.allow_expensive_queries: null}
+
+  ### Prefix
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            prefix:
+              text:
+                value: out
+
+  - match: { hits.total.value: 3 }
+
+  ### Fuzzy
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            fuzzy:
+              text:
+                value: outwide
+
+  - match: { hits.total.value: 3 }
+
+
+  ### Regexp
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            regexp:
+              text:
+                value: .*ou.*id.*
+
+  - match: { hits.total.value: 3 }
+
+  ### Wildcard
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            wildcard:
+              text:
+                value: out?ide
+
+  - match: { hits.total.value: 3 }
+
+  ### Range on text
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            range:
+              text:
+                gte: "theres"
+
+  - match: { hits.total.value: 2 }
+
+  ### Range on keyword
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            range:
+              text.raw:
+                gte : "Outside it is cold and wet"
+
+  - match: { hits.total.value: 2 }
+
+  ### Nested
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            nested:
+              path: "nested1"
+              query:
+                bool:
+                  must: [{"match": {"nested1.foo": "bar2"}}]
+
+  - match: { hits.total.value: 1 }
 
   ### Update setting to false
   - do:
@@ -145,3 +232,99 @@ teardown:
               query:
                 bool:
                   must: [{"match" : {"nested1.foo" : "bar2"}}]
+
+  ### Revert setting to true
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search.allow_expensive_queries: "true"
+        flat_settings: true
+
+  - match: {transient: {search.allow_expensive_queries: "true"}}
+
+  ### Prefix
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            prefix:
+              text:
+                value: out
+
+  - match: { hits.total.value: 3 }
+
+  ### Fuzzy
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            fuzzy:
+              text:
+                value: outwide
+
+  - match: { hits.total.value: 3 }
+
+  ### Regexp
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            regexp:
+              text:
+                value: .*ou.*id.*
+
+  - match: { hits.total.value: 3 }
+
+  ### Wildcard
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            wildcard:
+              text:
+                value: out?ide
+
+  - match: { hits.total.value: 3 }
+
+  ### Range on text
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            range:
+              text:
+                gte: "theres"
+
+  - match: { hits.total.value: 2 }
+
+  ### Range on keyword
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            range:
+              text.raw:
+                gte: "Outside it is cold and wet"
+
+  - match: { hits.total.value: 2 }
+
+  ### Nested
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            nested:
+              path: "nested1"
+              query:
+                bool:
+                  must: [{"match": {"nested1.foo": "bar2"}}]
+
+  - match: { hits.total.value: 1 }
diff --git a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 242bbfc9bd604..0000000000000
--- a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0365c37a03123ee8e30f75e44a1cb7d5ddd2fc52
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..54cf1bac15e42
--- /dev/null
+++ b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+1963afb27f340df8fc304d377971424832f4ce1a
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 056fe4a00cbb5..0000000000000
--- a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d56b30f75b2df92da8c6c0965ce72e7abb86347b
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..3297020fb5caa
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+fdff4122e8b8a2dbbc9de24be6963e7d7e33b794
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index fe30b9975cab4..0000000000000
--- a/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..3fd9e819ce181
--- /dev/null
+++ b/server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+ca406661129d35008411365d2b6e747dc39378af
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index c3ef488e826f8..0000000000000
--- a/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e56cc12d2f77d82946299b66f3416f9e621b2f3
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..11adaa175d1af
--- /dev/null
+++ b/server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+db053d5861406393254c28f6e46767879b504bb3
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index cfa7fa961780b..0000000000000
--- a/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5cddb5b65e7ead641483dcc2ffb0e50ad8d26eb7
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..99a2bbe095ea3
--- /dev/null
+++ b/server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+f5520ee7145f5d1ef02c7dc87483255d81b5bc6c
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 94a18fd3c3913..0000000000000
--- a/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cfabaedd80fe600cc7fda5ee12d90927fa96d87c
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..e05ece509f26e
--- /dev/null
+++ b/server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+2d81c0a3473cc865e7c4858890b7fbfb869bfbf8
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index ae7da5c259f69..0000000000000
--- a/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e90fbcc53531978fc03ef847ba396d4cdd89c7e4
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..2d8258caac8ab
--- /dev/null
+++ b/server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+6d009afeb485307dce111afb8bb157ebbbb0f212
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 8f6c2fe6d63c1..0000000000000
--- a/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd0b4cef132a50b3fa919f214a5316fcc78c46ea
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..0b228e2a8fa12
--- /dev/null
+++ b/server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+3a62908ec9eb6e826a56e697322c4c6b6c9a8573
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 4b2a346309f3f..0000000000000
--- a/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c90cc35089afc3f7802668c3969b5e7391b6d15a
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..c159ab983010d
--- /dev/null
+++ b/server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+d71d54ed6e0cf482ce16cf4f419441d83f646827
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 3b5ec91e4f938..0000000000000
--- a/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e324233cb8f069e4f6abcbab47368a83c3696f36
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..10be987e9c169
--- /dev/null
+++ b/server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+181915a7d21b73dff16591b20cdee22648e4181f
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 58d7518e7aefc..0000000000000
--- a/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-985a451e5f564c84271419a446e044ab589d6f22
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..fb143f9c9c3bf
--- /dev/null
+++ b/server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+d3d0bb76d9f4a5368d286a934615dbca7703b3d8
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index f51b876ad5fdf..0000000000000
--- a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-beff7cafe0fa5330b9b915825b69321faf0fcaa9
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..fd23c6f7e0389
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+95b9fd35e91a34c090ecf301d4dc29cabd198e6f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 736c206ba1b48..0000000000000
--- a/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b9256d3a2a64d79435a4c726af8a3c28c2b77d7f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..d4fc63b96d40f
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+1c8da46c3a172830372dfc23e18e9151bb14562c
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 586728768c5b4..0000000000000
--- a/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f38949db273a910e94a57229db2d8f3e4aef5e1f
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..51bc432029f38
--- /dev/null
+++ b/server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+b3ad5d3476ed85a529892962d057518555ccfcc9
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/lucene/queries/XIntervals.java b/server/src/main/java/org/apache/lucene/queries/XIntervals.java
new file mode 100644
index 0000000000000..2de278e3037ce
--- /dev/null
+++ b/server/src/main/java/org/apache/lucene/queries/XIntervals.java
@@ -0,0 +1,861 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.queries;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queries.intervals.IntervalIterator;
+import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
+import org.apache.lucene.queries.intervals.IntervalQuery;
+import org.apache.lucene.queries.intervals.Intervals;
+import org.apache.lucene.queries.intervals.IntervalsSource;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.MatchesIterator;
+import org.apache.lucene.search.MatchesUtils;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Replacement for {@link Intervals#wildcard(BytesRef)} and {@link Intervals#prefix(BytesRef)}
+ * until LUCENE-9050 is merged
+ */
+public final class XIntervals {
+
+    private XIntervals() {}
+
+    public static IntervalsSource wildcard(BytesRef wildcard) {
+        CompiledAutomaton ca = new CompiledAutomaton(WildcardQuery.toAutomaton(new Term("", wildcard)));
+        return new MultiTermIntervalsSource(ca, 128, wildcard.utf8ToString());
+    }
+
+    public static IntervalsSource prefix(BytesRef prefix) {
+        CompiledAutomaton ca = new CompiledAutomaton(PrefixQuery.toAutomaton(prefix));
+        return new MultiTermIntervalsSource(ca, 128, prefix.utf8ToString());
+    }
+
+    public static IntervalsSource multiterm(CompiledAutomaton ca, String label) {
+        return new MultiTermIntervalsSource(ca, 128, label);
+    }
+
+    static class MultiTermIntervalsSource extends IntervalsSource {
+
+        private final CompiledAutomaton automaton;
+        private final int maxExpansions;
+        private final String pattern;
+
+        MultiTermIntervalsSource(CompiledAutomaton automaton, int maxExpansions, String pattern) {
+            this.automaton = automaton;
+            if (maxExpansions > BooleanQuery.getMaxClauseCount()) {
+                throw new IllegalArgumentException("maxExpansions [" + maxExpansions
+                    + "] cannot be greater than BooleanQuery.getMaxClauseCount [" + BooleanQuery.getMaxClauseCount() + "]");
+            }
+            this.maxExpansions = maxExpansions;
+            this.pattern = pattern;
+        }
+
+        @Override
+        public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
+            Terms terms = ctx.reader().terms(field);
+            if (terms == null) {
+                return null;
+            }
+            List<IntervalIterator> subSources = new ArrayList<>();
+            TermsEnum te = automaton.getTermsEnum(terms);
+            BytesRef term;
+            int count = 0;
+            while ((term = te.next()) != null) {
+                subSources.add(TermIntervalsSource.intervals(term, te));
+                if (++count > maxExpansions) {
+                    throw new IllegalStateException("Automaton [" + this.pattern + "] expanded to too many terms (limit "
+                        + maxExpansions + ")");
+                }
+            }
+            if (subSources.size() == 0) {
+                return null;
+            }
+            return new DisjunctionIntervalIterator(subSources);
+        }
+
+        @Override
+        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
+            Terms terms = ctx.reader().terms(field);
+            if (terms == null) {
+                return null;
+            }
+            List<MatchesIterator> subMatches = new ArrayList<>();
+            TermsEnum te = automaton.getTermsEnum(terms);
+            BytesRef term;
+            int count = 0;
+            while ((term = te.next()) != null) {
+                MatchesIterator mi = XIntervals.TermIntervalsSource.matches(te, doc);
+                if (mi != null) {
+                    subMatches.add(mi);
+                    if (count++ > maxExpansions) {
+                        throw new IllegalStateException("Automaton " + term + " expanded to too many terms (limit " + maxExpansions + ")");
+                    }
+                }
+            }
+            MatchesIterator mi = MatchesUtils.disjunction(subMatches);
+            if (mi == null) {
+                return null;
+            }
+            return new IntervalMatchesIterator() {
+                @Override
+                public int gaps() {
+                    return 0;
+                }
+
+                @Override
+                public int width() {
+                    return 1;
+                }
+
+                @Override
+                public boolean next() throws IOException {
+                    return mi.next();
+                }
+
+                @Override
+                public int startPosition() {
+                    return mi.startPosition();
+                }
+
+                @Override
+                public int endPosition() {
+                    return mi.endPosition();
+                }
+
+                @Override
+                public int startOffset() throws IOException {
+                    return mi.startOffset();
+                }
+
+                @Override
+                public int endOffset() throws IOException {
+                    return mi.endOffset();
+                }
+
+                @Override
+                public MatchesIterator getSubMatches() throws IOException {
+                    return mi.getSubMatches();
+                }
+
+                @Override
+                public Query getQuery() {
+                    return mi.getQuery();
+                }
+            };
+        }
+
+        @Override
+        public void visit(String field, QueryVisitor visitor) {
+            visitor.visitLeaf(new IntervalQuery(field, this));
+        }
+
+        @Override
+        public int minExtent() {
+            return 1;
+        }
+
+        @Override
+        public Collection<IntervalsSource> pullUpDisjunctions() {
+            return Collections.singleton(this);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            MultiTermIntervalsSource that = (MultiTermIntervalsSource) o;
+            return maxExpansions == that.maxExpansions &&
+                Objects.equals(automaton, that.automaton) &&
+                Objects.equals(pattern, that.pattern);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(automaton, maxExpansions, pattern);
+        }
+
+        @Override
+        public String toString() {
+            return "MultiTerm(" + pattern + ")";
+        }
+    }
+
+    static class DisiWrapper {
+
+        public final DocIdSetIterator iterator;
+        public final IntervalIterator intervals;
+        public final long cost;
+        public final float matchCost; // the match cost for two-phase iterators, 0 otherwise
+        public int doc; // the current doc, used for comparison
+        public DisiWrapper next; // reference to a next element, see #topList
+
+        // An approximation of the iterator, or the iterator itself if it does not
+        // support two-phase iteration
+        public final DocIdSetIterator approximation;
+
+        DisiWrapper(IntervalIterator iterator) {
+            this.intervals = iterator;
+            this.iterator = iterator;
+            this.cost = iterator.cost();
+            this.doc = -1;
+            this.approximation = iterator;
+            this.matchCost = iterator.matchCost();
+        }
+
+    }
+
+    static final class DisiPriorityQueue implements Iterable<DisiWrapper> {
+
+        static int leftNode(int node) {
+            return ((node + 1) << 1) - 1;
+        }
+
+        static int rightNode(int leftNode) {
+            return leftNode + 1;
+        }
+
+        static int parentNode(int node) {
+            return ((node + 1) >>> 1) - 1;
+        }
+
+        private final DisiWrapper[] heap;
+        private int size;
+
+        DisiPriorityQueue(int maxSize) {
+            heap = new DisiWrapper[maxSize];
+            size = 0;
+        }
+
+        public int size() {
+            return size;
+        }
+
+        public DisiWrapper top() {
+            return heap[0];
+        }
+
+        /** Get the list of scorers which are on the current doc. */
+        DisiWrapper topList() {
+            final DisiWrapper[] heap = this.heap;
+            final int size = this.size;
+            DisiWrapper list = heap[0];
+            list.next = null;
+            if (size >= 3) {
+                list = topList(list, heap, size, 1);
+                list = topList(list, heap, size, 2);
+            } else if (size == 2 && heap[1].doc == list.doc) {
+                list = prepend(heap[1], list);
+            }
+            return list;
+        }
+
+        // prepend w1 (iterator) to w2 (list)
+        private DisiWrapper prepend(DisiWrapper w1, DisiWrapper w2) {
+            w1.next = w2;
+            return w1;
+        }
+
+        private DisiWrapper topList(DisiWrapper list, DisiWrapper[] heap,
+                                    int size, int i) {
+            final DisiWrapper w = heap[i];
+            if (w.doc == list.doc) {
+                list = prepend(w, list);
+                final int left = leftNode(i);
+                final int right = left + 1;
+                if (right < size) {
+                    list = topList(list, heap, size, left);
+                    list = topList(list, heap, size, right);
+                } else if (left < size && heap[left].doc == list.doc) {
+                    list = prepend(heap[left], list);
+                }
+            }
+            return list;
+        }
+
+        public DisiWrapper add(DisiWrapper entry) {
+            final DisiWrapper[] heap = this.heap;
+            final int size = this.size;
+            heap[size] = entry;
+            upHeap(size);
+            this.size = size + 1;
+            return heap[0];
+        }
+
+        public DisiWrapper pop() {
+            final DisiWrapper[] heap = this.heap;
+            final DisiWrapper result = heap[0];
+            final int i = --size;
+            heap[0] = heap[i];
+            heap[i] = null;
+            downHeap(i);
+            return result;
+        }
+
+        DisiWrapper updateTop() {
+            downHeap(size);
+            return heap[0];
+        }
+
+        void upHeap(int i) {
+            final DisiWrapper node = heap[i];
+            final int nodeDoc = node.doc;
+            int j = parentNode(i);
+            while (j >= 0 && nodeDoc < heap[j].doc) {
+                heap[i] = heap[j];
+                i = j;
+                j = parentNode(j);
+            }
+            heap[i] = node;
+        }
+
+        void downHeap(int size) {
+            int i = 0;
+            final DisiWrapper node = heap[0];
+            int j = leftNode(i);
+            if (j < size) {
+                int k = rightNode(j);
+                if (k < size && heap[k].doc < heap[j].doc) {
+                    j = k;
+                }
+                if (heap[j].doc < node.doc) {
+                    do {
+                        heap[i] = heap[j];
+                        i = j;
+                        j = leftNode(i);
+                        k = rightNode(j);
+                        if (k < size && heap[k].doc < heap[j].doc) {
+                            j = k;
+                        }
+                    } while (j < size && heap[j].doc < node.doc);
+                    heap[i] = node;
+                }
+            }
+        }
+
+        @Override
+        public Iterator<DisiWrapper> iterator() {
+            return Arrays.asList(heap).subList(0, size).iterator();
+        }
+
+    }
+
+    static class DisjunctionDISIApproximation extends DocIdSetIterator {
+
+        final DisiPriorityQueue subIterators;
+        final long cost;
+
+        DisjunctionDISIApproximation(DisiPriorityQueue subIterators) {
+            this.subIterators = subIterators;
+            long cost = 0;
+            for (DisiWrapper w : subIterators) {
+                cost += w.cost;
+            }
+            this.cost = cost;
+        }
+
+        @Override
+        public long cost() {
+            return cost;
+        }
+
+        @Override
+        public int docID() {
+            return subIterators.top().doc;
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            DisiWrapper top = subIterators.top();
+            final int doc = top.doc;
+            do {
+                top.doc = top.approximation.nextDoc();
+                top = subIterators.updateTop();
+            } while (top.doc == doc);
+
+            return top.doc;
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            DisiWrapper top = subIterators.top();
+            do {
+                top.doc = top.approximation.advance(target);
+                top = subIterators.updateTop();
+            } while (top.doc < target);
+
+            return top.doc;
+        }
+    }
+
+    static class DisjunctionIntervalIterator extends IntervalIterator {
+
+        final DocIdSetIterator approximation;
+        final PriorityQueue<IntervalIterator> intervalQueue;
+        final DisiPriorityQueue disiQueue;
+        final List<IntervalIterator> iterators;
+        final float matchCost;
+
+        IntervalIterator current = EMPTY;
+
+        DisjunctionIntervalIterator(List<IntervalIterator> iterators) {
+            this.disiQueue = new DisiPriorityQueue(iterators.size());
+            for (IntervalIterator it : iterators) {
+                disiQueue.add(new DisiWrapper(it));
+            }
+            this.approximation = new DisjunctionDISIApproximation(disiQueue);
+            this.iterators = iterators;
+            this.intervalQueue = new PriorityQueue<>(iterators.size()) {
+                @Override
+                protected boolean lessThan(IntervalIterator a, IntervalIterator b) {
+                    return a.end() < b.end() || (a.end() == b.end() && a.start() >= b.start());
+                }
+            };
+            float costsum = 0;
+            for (IntervalIterator it : iterators) {
+                costsum += it.cost();
+            }
+            this.matchCost = costsum;
+        }
+
+        @Override
+        public float matchCost() {
+            return matchCost;
+        }
+
+        @Override
+        public int start() {
+            return current.start();
+        }
+
+        @Override
+        public int end() {
+            return current.end();
+        }
+
+        @Override
+        public int gaps() {
+            return current.gaps();
+        }
+
+        private void reset() throws IOException {
+            intervalQueue.clear();
+            for (DisiWrapper dw = disiQueue.topList(); dw != null; dw = dw.next) {
+                dw.intervals.nextInterval();
+                intervalQueue.add(dw.intervals);
+            }
+            current = EMPTY;
+        }
+
+        @Override
+        public int nextInterval() throws IOException {
+            if (current == EMPTY || current == EXHAUSTED) {
+                if (intervalQueue.size() > 0) {
+                    current = intervalQueue.top();
+                }
+                return current.start();
+            }
+            int start = current.start(), end = current.end();
+            while (intervalQueue.size() > 0 && contains(intervalQueue.top(), start, end)) {
+                IntervalIterator it = intervalQueue.pop();
+                if (it != null && it.nextInterval() != NO_MORE_INTERVALS) {
+                    intervalQueue.add(it);
+                }
+            }
+            if (intervalQueue.size() == 0) {
+                current = EXHAUSTED;
+                return NO_MORE_INTERVALS;
+            }
+            current = intervalQueue.top();
+            return current.start();
+        }
+
+        private boolean contains(IntervalIterator it, int start, int end) {
+            return start >= it.start() && start <= it.end() && end >= it.start() && end <= it.end();
+        }
+
+        @Override
+        public int docID() {
+            return approximation.docID();
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            int doc = approximation.nextDoc();
+            reset();
+            return doc;
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            int doc = approximation.advance(target);
+            reset();
+            return doc;
+        }
+
+        @Override
+        public long cost() {
+            return approximation.cost();
+        }
+    }
+
+    private static final IntervalIterator EMPTY = new IntervalIterator() {
+
+        @Override
+        public int docID() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int nextDoc() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int advance(int target) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public long cost() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int start() {
+            return -1;
+        }
+
+        @Override
+        public int end() {
+            return -1;
+        }
+
+        @Override
+        public int gaps() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int nextInterval() {
+            return NO_MORE_INTERVALS;
+        }
+
+        @Override
+        public float matchCost() {
+            return 0;
+        }
+    };
+
+    private static final IntervalIterator EXHAUSTED = new IntervalIterator() {
+
+        @Override
+        public int docID() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int nextDoc() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int advance(int target) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public long cost() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int start() {
+            return NO_MORE_INTERVALS;
+        }
+
+        @Override
+        public int end() {
+            return NO_MORE_INTERVALS;
+        }
+
+        @Override
+        public int gaps() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int nextInterval() {
+            return NO_MORE_INTERVALS;
+        }
+
+        @Override
+        public float matchCost() {
+            return 0;
+        }
+    };
+
+    static class TermIntervalsSource extends IntervalsSource {
+
+        final BytesRef term;
+
+        TermIntervalsSource(BytesRef term) {
+            this.term = term;
+        }
+
+        @Override
+        public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
+            Terms terms = ctx.reader().terms(field);
+            if (terms == null)
+                return null;
+            if (terms.hasPositions() == false) {
+                throw new IllegalArgumentException("Cannot create an IntervalIterator over field " + field
+                    + " because it has no indexed positions");
+            }
+            TermsEnum te = terms.iterator();
+            if (te.seekExact(term) == false) {
+                return null;
+            }
+            return intervals(term, te);
+        }
+
+        static IntervalIterator intervals(BytesRef term, TermsEnum te) throws IOException {
+            PostingsEnum pe = te.postings(null, PostingsEnum.POSITIONS);
+            float cost = termPositionsCost(te);
+            return new IntervalIterator() {
+
+                @Override
+                public int docID() {
+                    return pe.docID();
+                }
+
+                @Override
+                public int nextDoc() throws IOException {
+                    int doc = pe.nextDoc();
+                    reset();
+                    return doc;
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    int doc = pe.advance(target);
+                    reset();
+                    return doc;
+                }
+
+                @Override
+                public long cost() {
+                    return pe.cost();
+                }
+
+                int pos = -1, upto;
+
+                @Override
+                public int start() {
+                    return pos;
+                }
+
+                @Override
+                public int end() {
+                    return pos;
+                }
+
+                @Override
+                public int gaps() {
+                    return 0;
+                }
+
+                @Override
+                public int nextInterval() throws IOException {
+                    if (upto <= 0)
+                        return pos = NO_MORE_INTERVALS;
+                    upto--;
+                    return pos = pe.nextPosition();
+                }
+
+                @Override
+                public float matchCost() {
+                    return cost;
+                }
+
+                private void reset() throws IOException {
+                    if (pe.docID() == NO_MORE_DOCS) {
+                        upto = -1;
+                        pos = NO_MORE_INTERVALS;
+                    }
+                    else {
+                        upto = pe.freq();
+                        pos = -1;
+                    }
+                }
+
+                @Override
+                public String toString() {
+                    return term.utf8ToString() + ":" + super.toString();
+                }
+            };
+        }
+
+        @Override
+        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
+            Terms terms = ctx.reader().terms(field);
+            if (terms == null)
+                return null;
+            if (terms.hasPositions() == false) {
+                throw new IllegalArgumentException("Cannot create an IntervalIterator over field " + field
+                    + " because it has no indexed positions");
+            }
+            TermsEnum te = terms.iterator();
+            if (te.seekExact(term) == false) {
+                return null;
+            }
+            return matches(te, doc);
+        }
+
+        static IntervalMatchesIterator matches(TermsEnum te, int doc) throws IOException {
+            PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS);
+            if (pe.advance(doc) != doc) {
+                return null;
+            }
+            return new IntervalMatchesIterator() {
+
+                @Override
+                public int gaps() {
+                    return 0;
+                }
+
+                @Override
+                public int width() {
+                    return 1;
+                }
+
+                int upto = pe.freq();
+                int pos = -1;
+
+                @Override
+                public boolean next() throws IOException {
+                    if (upto <= 0) {
+                        pos = IntervalIterator.NO_MORE_INTERVALS;
+                        return false;
+                    }
+                    upto--;
+                    pos = pe.nextPosition();
+                    return true;
+                }
+
+                @Override
+                public int startPosition() {
+                    return pos;
+                }
+
+                @Override
+                public int endPosition() {
+                    return pos;
+                }
+
+                @Override
+                public int startOffset() throws IOException {
+                    return pe.startOffset();
+                }
+
+                @Override
+                public int endOffset() throws IOException {
+                    return pe.endOffset();
+                }
+
+                @Override
+                public MatchesIterator getSubMatches() {
+                    return null;
+                }
+
+                @Override
+                public Query getQuery() {
+                    throw new UnsupportedOperationException();
+                }
+            };
+        }
+
+        @Override
+        public int minExtent() {
+            return 1;
+        }
+
+        @Override
+        public Collection<IntervalsSource> pullUpDisjunctions() {
+            return Collections.singleton(this);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(term);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            TermIntervalsSource that = (TermIntervalsSource) o;
+            return Objects.equals(term, that.term);
+        }
+
+        @Override
+        public String toString() {
+            return term.utf8ToString();
+        }
+
+        @Override
+        public void visit(String field, QueryVisitor visitor) {
+            visitor.consumeTerms(new IntervalQuery(field, this), new Term(field, term));
+        }
+
+        private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
+
+        private static final int TERM_OPS_PER_POS = 7;
+
+        static float termPositionsCost(TermsEnum termsEnum) throws IOException {
+            int docFreq = termsEnum.docFreq();
+            assert docFreq > 0;
+            long totalTermFreq = termsEnum.totalTermFreq();
+            float expOccurrencesInMatchingDoc = totalTermFreq / (float) docFreq;
+            return TERM_POSNS_SEEK_OPS_PER_DOC + expOccurrencesInMatchingDoc * TERM_OPS_PER_POS;
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
index 6f33f75692294..4ea78a17fd0e9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
@@ -286,7 +286,7 @@ enum Metrics {
         OS("os"),
         PROCESS("process"),
         JVM("jvm"),
-        THREAD_POOL("thread_pool"),
+        THREAD_POOL("threadPool"),
         TRANSPORT("transport"),
         HTTP("http"),
         PLUGINS("plugins"),
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index 4524a2c082c65..205404a564aea 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -19,17 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.node.stats;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 /**
  * A request to get node (cluster) level stats.
@@ -37,7 +32,18 @@
 public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
 
     private CommonStatsFlags indices = new CommonStatsFlags();
-    private final Set<String> requestedMetrics = new HashSet<>();
+    private boolean os;
+    private boolean process;
+    private boolean jvm;
+    private boolean threadPool;
+    private boolean fs;
+    private boolean transport;
+    private boolean http;
+    private boolean breaker;
+    private boolean script;
+    private boolean discovery;
+    private boolean ingest;
+    private boolean adaptiveSelection;
 
     public NodesStatsRequest() {
         super((String[]) null);
@@ -45,25 +51,19 @@ public NodesStatsRequest() {
 
     public NodesStatsRequest(StreamInput in) throws IOException {
         super(in);
-
         indices = new CommonStatsFlags(in);
-        requestedMetrics.clear();
-        if (in.getVersion().before(Version.V_7_7_0)) {
-            addOrRemoveMetric(in.readBoolean(), Metric.OS.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.PROCESS.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.JVM.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.THREAD_POOL.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.FS.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.TRANSPORT.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.HTTP.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.BREAKER.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.SCRIPT.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.DISCOVERY.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.INGEST.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metric.ADAPTIVE_SELECTION.metricName());
-        } else {
-            requestedMetrics.addAll(in.readStringList());
-        }
+        os = in.readBoolean();
+        process = in.readBoolean();
+        jvm = in.readBoolean();
+        threadPool = in.readBoolean();
+        fs = in.readBoolean();
+        transport = in.readBoolean();
+        http = in.readBoolean();
+        breaker = in.readBoolean();
+        script = in.readBoolean();
+        discovery = in.readBoolean();
+        ingest = in.readBoolean();
+        adaptiveSelection = in.readBoolean();
     }
 
     /**
@@ -79,7 +79,18 @@ public NodesStatsRequest(String... nodesIds) {
      */
     public NodesStatsRequest all() {
         this.indices.all();
-        this.requestedMetrics.addAll(Metric.allMetrics());
+        this.os = true;
+        this.process = true;
+        this.jvm = true;
+        this.threadPool = true;
+        this.fs = true;
+        this.transport = true;
+        this.http = true;
+        this.breaker = true;
+        this.script = true;
+        this.discovery = true;
+        this.ingest = true;
+        this.adaptiveSelection = true;
         return this;
     }
 
@@ -88,7 +99,18 @@ public NodesStatsRequest all() {
      */
     public NodesStatsRequest clear() {
         this.indices.clear();
-        this.requestedMetrics.clear();
+        this.os = false;
+        this.process = false;
+        this.jvm = false;
+        this.threadPool = false;
+        this.fs = false;
+        this.transport = false;
+        this.http = false;
+        this.breaker = false;
+        this.script = false;
+        this.discovery = false;
+        this.ingest = false;
+        this.adaptiveSelection = false;
         return this;
     }
 
@@ -117,14 +139,14 @@ public NodesStatsRequest indices(boolean indices) {
      * Should the node OS be returned.
      */
     public boolean os() {
-        return Metric.OS.containedIn(requestedMetrics);
+        return this.os;
     }
 
     /**
      * Should the node OS be returned.
      */
     public NodesStatsRequest os(boolean os) {
-        addOrRemoveMetric(os, Metric.OS.metricName());
+        this.os = os;
         return this;
     }
 
@@ -132,14 +154,14 @@ public NodesStatsRequest os(boolean os) {
      * Should the node Process be returned.
      */
     public boolean process() {
-        return Metric.PROCESS.containedIn(requestedMetrics);
+        return this.process;
     }
 
     /**
      * Should the node Process be returned.
      */
     public NodesStatsRequest process(boolean process) {
-        addOrRemoveMetric(process, Metric.PROCESS.metricName());
+        this.process = process;
         return this;
     }
 
@@ -147,14 +169,14 @@ public NodesStatsRequest process(boolean process) {
      * Should the node JVM be returned.
      */
     public boolean jvm() {
-        return Metric.JVM.containedIn(requestedMetrics);
+        return this.jvm;
     }
 
     /**
      * Should the node JVM be returned.
      */
     public NodesStatsRequest jvm(boolean jvm) {
-        addOrRemoveMetric(jvm, Metric.JVM.metricName());
+        this.jvm = jvm;
         return this;
     }
 
@@ -162,14 +184,14 @@ public NodesStatsRequest jvm(boolean jvm) {
      * Should the node Thread Pool be returned.
      */
     public boolean threadPool() {
-        return Metric.THREAD_POOL.containedIn(requestedMetrics);
+        return this.threadPool;
     }
 
     /**
      * Should the node Thread Pool be returned.
      */
     public NodesStatsRequest threadPool(boolean threadPool) {
-        addOrRemoveMetric(threadPool, Metric.THREAD_POOL.metricName());
+        this.threadPool = threadPool;
         return this;
     }
 
@@ -177,14 +199,14 @@ public NodesStatsRequest threadPool(boolean threadPool) {
      * Should the node file system stats be returned.
      */
     public boolean fs() {
-        return Metric.FS.containedIn(requestedMetrics);
+        return this.fs;
     }
 
     /**
      * Should the node file system stats be returned.
      */
     public NodesStatsRequest fs(boolean fs) {
-        addOrRemoveMetric(fs, Metric.FS.metricName());
+        this.fs = fs;
         return this;
     }
 
@@ -192,14 +214,14 @@ public NodesStatsRequest fs(boolean fs) {
      * Should the node Transport be returned.
      */
     public boolean transport() {
-        return Metric.TRANSPORT.containedIn(requestedMetrics);
+        return this.transport;
     }
 
     /**
      * Should the node Transport be returned.
      */
     public NodesStatsRequest transport(boolean transport) {
-        addOrRemoveMetric(transport, Metric.TRANSPORT.metricName());
+        this.transport = transport;
         return this;
     }
 
@@ -207,144 +229,90 @@ public NodesStatsRequest transport(boolean transport) {
      * Should the node HTTP be returned.
      */
     public boolean http() {
-        return Metric.HTTP.containedIn(requestedMetrics);
+        return this.http;
     }
 
     /**
      * Should the node HTTP be returned.
      */
     public NodesStatsRequest http(boolean http) {
-        addOrRemoveMetric(http, Metric.HTTP.metricName());
+        this.http = http;
         return this;
     }
 
     public boolean breaker() {
-        return Metric.BREAKER.containedIn(requestedMetrics);
+        return this.breaker;
     }
 
     /**
      * Should the node's circuit breaker stats be returned.
      */
     public NodesStatsRequest breaker(boolean breaker) {
-        addOrRemoveMetric(breaker, Metric.BREAKER.metricName());
+        this.breaker = breaker;
         return this;
     }
 
     public boolean script() {
-        return Metric.SCRIPT.containedIn(requestedMetrics);
+        return script;
     }
 
     public NodesStatsRequest script(boolean script) {
-        addOrRemoveMetric(script, Metric.SCRIPT.metricName());
+        this.script = script;
         return this;
     }
 
 
     public boolean discovery() {
-        return Metric.DISCOVERY.containedIn(requestedMetrics);
+        return this.discovery;
     }
 
     /**
      * Should the node's discovery stats be returned.
      */
     public NodesStatsRequest discovery(boolean discovery) {
-        addOrRemoveMetric(discovery, Metric.DISCOVERY.metricName());
+        this.discovery = discovery;
         return this;
     }
 
     public boolean ingest() {
-        return Metric.INGEST.containedIn(requestedMetrics);
+        return ingest;
     }
 
     /**
      * Should ingest statistics be returned.
      */
     public NodesStatsRequest ingest(boolean ingest) {
-        addOrRemoveMetric(ingest, Metric.INGEST.metricName());
+        this.ingest = ingest;
         return this;
     }
 
     public boolean adaptiveSelection() {
-        return Metric.ADAPTIVE_SELECTION.containedIn(requestedMetrics);
+        return adaptiveSelection;
     }
 
     /**
      * Should adaptiveSelection statistics be returned.
      */
     public NodesStatsRequest adaptiveSelection(boolean adaptiveSelection) {
-        addOrRemoveMetric(adaptiveSelection, Metric.ADAPTIVE_SELECTION.metricName());
+        this.adaptiveSelection = adaptiveSelection;
         return this;
     }
 
-    /**
-     * Helper method for adding and removing metrics.
-     * @param includeMetric Whether or not to include a metric.
-     * @param metricName Name of the metric to include or remove.
-     */
-    private void addOrRemoveMetric(boolean includeMetric, String metricName) {
-        if (includeMetric) {
-            requestedMetrics.add(metricName);
-        } else {
-            requestedMetrics.remove(metricName);
-        }
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         indices.writeTo(out);
-        if (out.getVersion().before(Version.V_7_7_0)) {
-            out.writeBoolean(Metric.OS.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.PROCESS.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.JVM.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.THREAD_POOL.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.FS.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.TRANSPORT.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.HTTP.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.BREAKER.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.SCRIPT.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.DISCOVERY.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.INGEST.containedIn(requestedMetrics));
-            out.writeBoolean(Metric.ADAPTIVE_SELECTION.containedIn(requestedMetrics));
-        } else {
-            out.writeStringArray(requestedMetrics.toArray(String[]::new));
-        }
-    }
-
-    /**
-     * An enumeration of the "core" sections of metrics that may be requested
-     * from the nodes stats endpoint. Eventually this list will be pluggable.
-     */
-    private enum Metric {
-        OS("os"),
-        PROCESS("process"),
-        JVM("jvm"),
-        THREAD_POOL("thread_pool"),
-        FS("fs"),
-        TRANSPORT("transport"),
-        HTTP("http"),
-        BREAKER("breaker"),
-        SCRIPT("script"),
-        DISCOVERY("discovery"),
-        INGEST("ingest"),
-        ADAPTIVE_SELECTION("adaptiveSelection");
-
-        private String metricName;
-
-        Metric(String name) {
-            this.metricName = name;
-        }
-
-        String metricName() {
-            return this.metricName;
-        }
-
-        boolean containedIn(Set<String> metricNames) {
-            return metricNames.contains(this.metricName());
-        }
-
-        static Set<String> allMetrics() {
-            return Arrays.stream(values()).map(Metric::metricName).collect(Collectors.toSet());
-        }
+        out.writeBoolean(os);
+        out.writeBoolean(process);
+        out.writeBoolean(jvm);
+        out.writeBoolean(threadPool);
+        out.writeBoolean(fs);
+        out.writeBoolean(transport);
+        out.writeBoolean(http);
+        out.writeBoolean(breaker);
+        out.writeBoolean(script);
+        out.writeBoolean(discovery);
+        out.writeBoolean(ingest);
+        out.writeBoolean(adaptiveSelection);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 5984c7276add7..0d2cac9b401dd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.admin.cluster.snapshots.restore;
 
 import org.elasticsearch.ElasticsearchGenerationException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -41,9 +40,9 @@
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 
 /**
@@ -61,6 +60,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private boolean includeGlobalState = false;
     private boolean partial = false;
     private boolean includeAliases = true;
+    private Settings settings = EMPTY_SETTINGS;
     private Settings indexSettings = EMPTY_SETTINGS;
     private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
 
@@ -90,9 +90,7 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException {
         includeGlobalState = in.readBoolean();
         partial = in.readBoolean();
         includeAliases = in.readBoolean();
-        if (in.getVersion().before(Version.V_7_7_0)) {
-            readSettingsFromStream(in); // formerly the unused settings field
-        }
+        settings = readSettingsFromStream(in);
         indexSettings = readSettingsFromStream(in);
         ignoreIndexSettings = in.readStringArray();
     }
@@ -110,9 +108,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(includeGlobalState);
         out.writeBoolean(partial);
         out.writeBoolean(includeAliases);
-        if (out.getVersion().before(Version.V_7_7_0)) {
-            writeSettingsToStream(Settings.EMPTY, out); // formerly the unused settings field
-        }
+        writeSettingsToStream(settings, out);
         writeSettingsToStream(indexSettings, out);
         out.writeStringArray(ignoreIndexSettings);
     }
@@ -132,6 +128,9 @@ public ActionRequestValidationException validate() {
         if (indicesOptions == null) {
             validationException = addValidationError("indicesOptions is missing", validationException);
         }
+        if (settings == null) {
+            validationException = addValidationError("settings are missing", validationException);
+        }
         if (indexSettings == null) {
             validationException = addValidationError("indexSettings are missing", validationException);
         }
@@ -325,6 +324,74 @@ public RestoreSnapshotRequest partial(boolean partial) {
         return this;
     }
 
+    /**
+     * Sets repository-specific restore settings.
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param settings repository-specific snapshot settings
+     * @return this request
+     */
+    public RestoreSnapshotRequest settings(Settings settings) {
+        this.settings = settings;
+        return this;
+    }
+
+    /**
+     * Sets repository-specific restore settings.
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param settings repository-specific snapshot settings
+     * @return this request
+     */
+    public RestoreSnapshotRequest settings(Settings.Builder settings) {
+        this.settings = settings.build();
+        return this;
+    }
+
+    /**
+     * Sets repository-specific restore settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public RestoreSnapshotRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
+    /**
+     * Sets repository-specific restore settings
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @return this request
+     */
+    public RestoreSnapshotRequest settings(Map<String, Object> source) {
+        try {
+            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+            builder.map(source);
+            settings(Strings.toString(builder), builder.contentType());
+        } catch (IOException e) {
+            throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
+        }
+        return this;
+    }
+
+    /**
+     * Returns repository-specific restore settings
+     *
+     * @return restore settings
+     */
+    public Settings settings() {
+        return this.settings;
+    }
+
     /**
      * Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
      */
@@ -455,6 +522,11 @@ public RestoreSnapshotRequest source(Map<String, Object> source) {
                 }
             } else if (name.equals("partial")) {
                 partial(nodeBooleanValue(entry.getValue(), "partial"));
+            } else if (name.equals("settings")) {
+                if (!(entry.getValue() instanceof Map)) {
+                    throw new IllegalArgumentException("malformed settings section");
+                }
+                settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("include_global_state")) {
                 includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state");
             } else if (name.equals("include_aliases")) {
@@ -514,6 +586,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("include_global_state", includeGlobalState);
         builder.field("partial", partial);
         builder.field("include_aliases", includeAliases);
+        if (settings != null) {
+            builder.startObject("settings");
+            if (settings.isEmpty() == false) {
+                settings.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
         if (indexSettings != null) {
             builder.startObject("index_settings");
             if (indexSettings.isEmpty() == false) {
@@ -550,6 +629,7 @@ public boolean equals(Object o) {
             Objects.equals(indicesOptions, that.indicesOptions) &&
             Objects.equals(renamePattern, that.renamePattern) &&
             Objects.equals(renameReplacement, that.renameReplacement) &&
+            Objects.equals(settings, that.settings) &&
             Objects.equals(indexSettings, that.indexSettings) &&
             Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings);
     }
@@ -557,7 +637,7 @@ public boolean equals(Object o) {
     @Override
     public int hashCode() {
         int result = Objects.hash(snapshot, repository, indicesOptions, renamePattern, renameReplacement, waitForCompletion,
-            includeGlobalState, partial, includeAliases, indexSettings);
+            includeGlobalState, partial, includeAliases, settings, indexSettings);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(ignoreIndexSettings);
         return result;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
index 08d509ce3ac74..f530261644776 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
@@ -127,6 +127,60 @@ public RestoreSnapshotRequestBuilder setRenameReplacement(String renameReplaceme
         return this;
     }
 
+
+    /**
+     * Sets repository-specific restore settings.
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param settings repository-specific snapshot settings
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setSettings(Settings settings) {
+        request.settings(settings);
+        return this;
+    }
+
+    /**
+     * Sets repository-specific restore settings.
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param settings repository-specific snapshot settings
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setSettings(Settings.Builder settings) {
+        request.settings(settings);
+        return this;
+    }
+
+    /**
+     * Sets repository-specific restore settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets repository-specific restore settings
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setSettings(Map<String, Object> source) {
+        request.settings(source);
+        return this;
+    }
+
     /**
      * If this parameter is set to true the operation will wait for completion of restore process before returning.
      *
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
index a6299cbee1ab8..c596449c0e6b6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
@@ -19,11 +19,8 @@
 
 package org.elasticsearch.action.admin.indices.forcemerge;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -56,15 +53,6 @@ public static final class Defaults {
     private boolean onlyExpungeDeletes = Defaults.ONLY_EXPUNGE_DELETES;
     private boolean flush = Defaults.FLUSH;
 
-    private static final Version FORCE_MERGE_UUID_VERSION = Version.V_7_7_0;
-
-    /**
-     * Force merge UUID to store in the live commit data of a shard under
-     * {@link org.elasticsearch.index.engine.Engine#FORCE_MERGE_UUID_KEY} after force merging it.
-     */
-    @Nullable
-    private final String forceMergeUUID;
-
     /**
      * Constructs a merge request over one or more indices.
      *
@@ -72,7 +60,6 @@ public static final class Defaults {
      */
     public ForceMergeRequest(String... indices) {
         super(indices);
-        forceMergeUUID = UUIDs.randomBase64UUID();
     }
 
     public ForceMergeRequest(StreamInput in) throws IOException {
@@ -80,11 +67,6 @@ public ForceMergeRequest(StreamInput in) throws IOException {
         maxNumSegments = in.readInt();
         onlyExpungeDeletes = in.readBoolean();
         flush = in.readBoolean();
-        if (in.getVersion().onOrAfter(FORCE_MERGE_UUID_VERSION)) {
-            forceMergeUUID = in.readOptionalString();
-        } else {
-            forceMergeUUID = null;
-        }
     }
 
     /**
@@ -121,15 +103,6 @@ public ForceMergeRequest onlyExpungeDeletes(boolean onlyExpungeDeletes) {
         return this;
     }
 
-    /**
-     * Force merge UUID to use when force merging or {@code null} if not using one in a mixed version cluster containing nodes older than
-     * {@link #FORCE_MERGE_UUID_VERSION}.
-     */
-    @Nullable
-    public String forceMergeUUID() {
-        return forceMergeUUID;
-    }
-
     /**
      * Should flush be performed after the merge. Defaults to {@code true}.
      */
@@ -159,9 +132,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeInt(maxNumSegments);
         out.writeBoolean(onlyExpungeDeletes);
         out.writeBoolean(flush);
-        if (out.getVersion().onOrAfter(FORCE_MERGE_UUID_VERSION)) {
-            out.writeOptionalString(forceMergeUUID);
-        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
index 9d1e5756c81d6..751276812bddc 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
@@ -310,7 +310,9 @@ List<String> getIndices() {
 
         FieldCapabilities build(boolean withIndices) {
             final String[] indices;
-            Collections.sort(indiceList, Comparator.comparing(o -> o.name));
+            /* Eclipse can't deal with o -> o.name, maybe because of
+             * https://bugs.eclipse.org/bugs/show_bug.cgi?id=511750 */
+            Collections.sort(indiceList, Comparator.comparing((IndexCaps o) -> o.name));
             if (withIndices) {
                 indices = indiceList.stream()
                     .map(caps -> caps.name)
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
index bcb16c8690b04..09bba6eb0da07 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
@@ -69,13 +69,6 @@ public void execute(SimulatePipelineRequest.Parsed request, ActionListener<Simul
             final AtomicInteger counter = new AtomicInteger();
             final List<SimulateDocumentResult> responses =
                 new CopyOnWriteArrayList<>(new SimulateDocumentBaseResult[request.getDocuments().size()]);
-
-            if (request.getDocuments().isEmpty()) {
-                l.onResponse(new SimulatePipelineResponse(request.getPipeline().getId(),
-                    request.isVerbose(), responses));
-                return;
-            }
-
             int iter = 0;
             for (IngestDocument ingestDocument : request.getDocuments()) {
                 final int index = iter;
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
index ba51378cfaff8..949d32c8fe734 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -173,15 +173,8 @@ static Parsed parse(Map<String, Object> config, boolean verbose, IngestService i
     private static List<IngestDocument> parseDocs(Map<String, Object> config) {
         List<Map<String, Object>> docs =
             ConfigurationUtils.readList(null, null, config, Fields.DOCS);
-        if (docs.isEmpty()) {
-            throw new IllegalArgumentException("must specify at least one document in [docs]");
-        }
         List<IngestDocument> ingestDocumentList = new ArrayList<>();
-        for (Object object : docs) {
-            if ((object instanceof Map) ==  false) {
-                throw new IllegalArgumentException("malformed [docs] section, should include an inner object");
-            }
-            Map<String, Object> dataMap = (Map<String, Object>) object;
+        for (Map<String, Object> dataMap : docs) {
             Map<String, Object> document = ConfigurationUtils.readMap(null, null,
                 dataMap, Fields.SOURCE);
             String index = ConfigurationUtils.readStringOrIntProperty(null, null,
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 1d2339dd9e6fe..8caf615dc1d07 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.search.TransportSearchAction.SearchTimeProvider;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
@@ -78,7 +77,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
     private final BiFunction<String, String, Transport.Connection> nodeIdToConnection;
     private final SearchTask task;
     private final SearchPhaseResults<Result> results;
-    private final ClusterState clusterState;
+    private final long clusterStateVersion;
     private final Map<String, AliasFilter> aliasFilter;
     private final Map<String, Float> concreteIndexBoosts;
     private final Map<String, Set<String>> indexRoutings;
@@ -90,7 +89,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
     private final SearchTimeProvider timeProvider;
     private final SearchResponse.Clusters clusters;
 
-    protected final GroupShardsIterator<SearchShardIterator> toSkipShardsIts;
+    private final GroupShardsIterator<SearchShardIterator> toSkipShardsIts;
     protected final GroupShardsIterator<SearchShardIterator> shardsIts;
     private final int expectedTotalOps;
     private final AtomicInteger totalOps = new AtomicInteger();
@@ -99,14 +98,14 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
     private final boolean throttleConcurrentRequests;
 
     AbstractSearchAsyncAction(String name, Logger logger, SearchTransportService searchTransportService,
-                              BiFunction<String, String, Transport.Connection> nodeIdToConnection,
-                              Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
-                              Map<String, Set<String>> indexRoutings,
-                              Executor executor, SearchRequest request,
-                              ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
-                              SearchTimeProvider timeProvider, ClusterState clusterState,
-                              SearchTask task, SearchPhaseResults<Result> resultConsumer, int maxConcurrentRequestsPerNode,
-                              SearchResponse.Clusters clusters) {
+                                        BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                                        Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                                        Map<String, Set<String>> indexRoutings,
+                                        Executor executor, SearchRequest request,
+                                        ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
+                                        SearchTimeProvider timeProvider, long clusterStateVersion,
+                                        SearchTask task, SearchPhaseResults<Result> resultConsumer, int maxConcurrentRequestsPerNode,
+                                        SearchResponse.Clusters clusters) {
         super(name);
         final List<SearchShardIterator> toSkipIterators = new ArrayList<>();
         final List<SearchShardIterator> iterators = new ArrayList<>();
@@ -135,7 +134,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
         this.task = task;
         this.listener = listener;
         this.nodeIdToConnection = nodeIdToConnection;
-        this.clusterState = clusterState;
+        this.clusterStateVersion = clusterStateVersion;
         this.concreteIndexBoosts = concreteIndexBoosts;
         this.aliasFilter = aliasFilter;
         this.indexRoutings = indexRoutings;
@@ -339,7 +338,7 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha
                 final String resultsFrom = results.getSuccessfulResults()
                     .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(","));
                 logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
-                    currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterState.version());
+                    currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion);
             }
             executePhase(nextPhase);
         }
@@ -375,11 +374,6 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
         // we do make sure to clean it on a successful response from a shard
         SearchShardTarget shardTarget = shardIt.newSearchShardTarget(nodeId);
         onShardFailure(shardIndex, shardTarget, e);
-        final ShardRouting nextShard = shardIt.nextOrNull();
-        final boolean lastShard = nextShard == null;
-        if (lastShard) {
-            onShardGroupFailure(shardIndex, shardTarget, e);
-        }
 
         if (totalOps.incrementAndGet() == expectedTotalOps) {
             if (logger.isDebugEnabled()) {
@@ -390,8 +384,11 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
                     logger.trace(new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e);
                 }
             }
+            onShardGroupFailure(shardIndex, e);
             onPhaseDone();
         } else {
+            final ShardRouting nextShard = shardIt.nextOrNull();
+            final boolean lastShard = nextShard == null;
             // trace log this exception
             logger.trace(() -> new ParameterizedMessage(
                 "{}: Failed to execute [{}] lastShard [{}]",
@@ -407,6 +404,7 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
                             shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e);
                     }
                 }
+                onShardGroupFailure(shardIndex, e);
             }
         }
     }
@@ -414,11 +412,10 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
     /**
      * Executed once for every {@link ShardId} that failed on all available shard routing.
      *
-     * @param shardIndex the shard index that failed
-     * @param shardTarget the last shard target for this failure
-     * @param exc the last failure reason
+     * @param shardIndex the shard target that failed
+     * @param exc the final failure reason
      */
-    protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {}
+    protected void onShardGroupFailure(int shardIndex, Exception exc) {}
 
     /**
      * Executed once for every failed shard level request. This method is invoked before the next replica is tried for the given
@@ -562,7 +559,7 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) {
             try {
                 SearchShardTarget searchShardTarget = entry.getSearchShardTarget();
                 Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
-                sendReleaseSearchContext(entry.getContextId(), connection, searchShardTarget.getOriginalIndices());
+                sendReleaseSearchContext(entry.getRequestId(), connection, searchShardTarget.getOriginalIndices());
             } catch (Exception inner) {
                 inner.addSuppressed(exception);
                 logger.trace("failed to release context", inner);
@@ -684,8 +681,4 @@ private synchronized Runnable tryQueue(Runnable runnable) {
             return toExecute;
         }
     }
-
-    protected ClusterState clusterState() {
-        return clusterState;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
index 70eeb2a811e5b..9d8b159b8e014 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
@@ -21,7 +21,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.search.SearchService.CanMatchResponse;
@@ -32,9 +31,11 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.transport.Transport;
 
+import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
@@ -60,17 +61,17 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction<CanMa
     private final GroupShardsIterator<SearchShardIterator> shardsIts;
 
     CanMatchPreFilterSearchPhase(Logger logger, SearchTransportService searchTransportService,
-                                 BiFunction<String, String, Transport.Connection> nodeIdToConnection,
-                                 Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
-                                 Map<String, Set<String>> indexRoutings,
-                                 Executor executor, SearchRequest request,
-                                 ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
-                                 TransportSearchAction.SearchTimeProvider timeProvider, ClusterState clusterState,
-                                 SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory,
-                                 SearchResponse.Clusters clusters) {
+                                        BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                                        Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                                        Map<String, Set<String>> indexRoutings,
+                                        Executor executor, SearchRequest request,
+                                        ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
+                                        TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion,
+                                        SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory,
+                                        SearchResponse.Clusters clusters) {
         //We set max concurrent shard requests to the number of shards so no throttling happens for can_match requests
         super("can_match", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
-                executor, request, listener, shardsIts, timeProvider, clusterState, task,
+                executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task,
                 new CanMatchSearchPhaseResults(shardsIts.size()), shardsIts.size(), clusters);
         this.phaseFactory = phaseFactory;
         this.shardsIts = shardsIts;
@@ -126,18 +127,7 @@ private static List<SearchShardIterator> sortShards(GroupShardsIterator<SearchSh
     }
 
     private static boolean shouldSortShards(MinAndMax<?>[] minAndMaxes) {
-        Class<?> clazz = null;
-        for (MinAndMax<?> minAndMax : minAndMaxes) {
-            if (clazz == null) {
-                clazz = minAndMax == null ? null : minAndMax.getMin().getClass();
-            } else if (minAndMax != null && clazz != minAndMax.getMin().getClass()) {
-                // we don't support sort values that mix different types (e.g.: long/double, numeric/keyword).
-                // TODO: we could fail the request because there is a high probability
-                //  that the merging of topdocs will fail later for the same reason ?
-                return false;
-            }
-        }
-        return clazz != null;
+        return Arrays.stream(minAndMaxes).anyMatch(Objects::nonNull);
     }
 
     private static Comparator<Integer> shardComparator(GroupShardsIterator<SearchShardIterator> shardsIts,
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
index d0abf798501b3..c33eecee8bc75 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
@@ -111,7 +111,7 @@ void cleanScrollIds(List<ScrollIdForNode> parsedScrollIds) {
                     } else {
                         try {
                             Transport.Connection connection = searchTransportService.getConnection(target.getClusterAlias(), node);
-                            searchTransportService.sendFreeContext(connection, target.getContextId(),
+                            searchTransportService.sendFreeContext(connection, target.getScrollId(),
                                 ActionListener.wrap(freed -> onFreedContext(freed.isFreed()), e -> onFailedFreedContext(e, node)));
                         } catch (Exception e) {
                             onFailedFreedContext(e, node);
diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index 8352469042a58..b4d52fa418e10 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.query.QuerySearchRequest;
@@ -71,11 +72,13 @@ public void run() throws IOException {
         final CountedCollector<SearchPhaseResult> counter = new CountedCollector<>(queryResult::consumeResult,
             resultList.size(),
             () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context);
+        final SearchSourceBuilder sourceBuilder = context.getRequest().source();
+        progressListener.notifyListShards(progressListener.searchShards(resultList), sourceBuilder == null || sourceBuilder.size() != 0);
         for (final DfsSearchResult dfsResult : resultList) {
             final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget();
             Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
             QuerySearchRequest querySearchRequest = new QuerySearchRequest(searchShardTarget.getOriginalIndices(),
-                    dfsResult.getContextId(), dfs);
+                    dfsResult.getRequestId(), dfs);
             final int shardIndex = dfsResult.getShardIndex();
             searchTransportService.sendExecuteQuery(connection, querySearchRequest, context.getTask(),
                 new SearchActionListener<QuerySearchResult>(searchShardTarget, shardIndex) {
@@ -93,15 +96,14 @@ protected void innerOnResponse(QuerySearchResult response) {
                     public void onFailure(Exception exception) {
                         try {
                             context.getLogger().debug(() -> new ParameterizedMessage("[{}] Failed to execute query phase",
-                                querySearchRequest.contextId()), exception);
-                            progressListener.notifyQueryFailure(shardIndex, searchShardTarget, exception);
+                                querySearchRequest.id()), exception);
+                            progressListener.notifyQueryFailure(shardIndex, exception);
                             counter.onFailure(shardIndex, searchShardTarget, exception);
                         } finally {
                             // the query might not have been executed at all (for example because thread pool rejected
                             // execution) and the search context that was created in dfs phase might not be released.
                             // release it again to be in the safe side
-                            context.sendReleaseSearchContext(
-                                querySearchRequest.contextId(), connection, searchShardTarget.getOriginalIndices());
+                            context.sendReleaseSearchContext(querySearchRequest.id(), connection, searchShardTarget.getOriginalIndices());
                         }
                     }
                 });
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 9142053be91d2..41d216072e4b2 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -22,9 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.search.ScoreDoc;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
@@ -32,7 +30,6 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.Transport;
 
@@ -53,21 +50,17 @@ final class FetchSearchPhase extends SearchPhase {
     private final Logger logger;
     private final SearchPhaseResults<SearchPhaseResult> resultConsumer;
     private final SearchProgressListener progressListener;
-    private final ClusterState clusterState;
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
-                     SearchPhaseContext context,
-                     ClusterState clusterState) {
-        this(resultConsumer, searchPhaseController, context, clusterState,
+                     SearchPhaseContext context) {
+        this(resultConsumer, searchPhaseController, context,
             (response, scrollId) -> new ExpandSearchPhase(context, response, scrollId));
     }
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
-                     SearchPhaseContext context,
-                     ClusterState clusterState,
-                     BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory) {
+                     SearchPhaseContext context, BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory) {
         super("fetch");
         if (context.getNumShards() != resultConsumer.getNumShards()) {
             throw new IllegalStateException("number of shards must match the length of the query results but doesn't:"
@@ -81,7 +74,6 @@ final class FetchSearchPhase extends SearchPhase {
         this.logger = context.getLogger();
         this.resultConsumer = resultConsumer;
         this.progressListener = context.getTask().getProgressListener();
-        this.clusterState = clusterState;
     }
 
     @Override
@@ -105,14 +97,8 @@ public void onFailure(Exception e) {
     private void innerRun() throws IOException {
         final int numShards = context.getNumShards();
         final boolean isScrollSearch = context.getRequest().scroll() != null;
-        final List<SearchPhaseResult> phaseResults = queryResults.asList();
-        final String scrollId;
-        if (isScrollSearch) {
-            final boolean includeContextUUID = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0);
-            scrollId = TransportSearchHelper.buildScrollId(queryResults, includeContextUUID);
-        } else {
-            scrollId = null;
-        }
+        List<SearchPhaseResult> phaseResults = queryResults.asList();
+        String scrollId = isScrollSearch ? TransportSearchHelper.buildScrollId(queryResults) : null;
         final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = resultConsumer.reduce();
         final boolean queryAndFetchOptimization = queryResults.length() == 1;
         final Runnable finishPhase = ()
@@ -157,7 +143,7 @@ private void innerRun() throws IOException {
                         SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
                         Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(),
                             searchShardTarget.getNodeId());
-                        ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getContextId(), i, entry,
+                        ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getRequestId(), i, entry,
                             lastEmittedDocPerShard, searchShardTarget.getOriginalIndices());
                         executeFetch(i, searchShardTarget, counter, fetchSearchRequest, queryResult.queryResult(),
                             connection);
@@ -167,10 +153,10 @@ private void innerRun() throws IOException {
         }
     }
 
-    protected ShardFetchSearchRequest createFetchRequest(SearchContextId contextId, int index, IntArrayList entry,
-                                                         ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices) {
+    protected ShardFetchSearchRequest createFetchRequest(long queryId, int index, IntArrayList entry,
+                                                               ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices) {
         final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[index] : null;
-        return new ShardFetchSearchRequest(originalIndices, contextId, entry, lastEmittedDoc);
+        return new ShardFetchSearchRequest(originalIndices, queryId, entry, lastEmittedDoc);
     }
 
     private void executeFetch(final int shardIndex, final SearchShardTarget shardTarget,
@@ -192,8 +178,7 @@ public void innerOnResponse(FetchSearchResult result) {
                 @Override
                 public void onFailure(Exception e) {
                     try {
-                        logger.debug(
-                            () -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.contextId()), e);
+                        logger.debug(() -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.id()), e);
                         progressListener.notifyFetchFailure(shardIndex, e);
                         counter.onFailure(shardIndex, shardTarget, e);
                     } finally {
@@ -216,7 +201,7 @@ private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
             try {
                 SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
                 Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
-                context.sendReleaseSearchContext(queryResult.getContextId(), connection, searchShardTarget.getOriginalIndices());
+                context.sendReleaseSearchContext(queryResult.getRequestId(), connection, searchShardTarget.getOriginalIndices());
             } catch (Exception e) {
                 context.getLogger().trace("failed to release context", e);
             }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
index d69a10334bd78..18b61516897d1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
@@ -20,17 +20,16 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.search.internal.SearchContextId;
 
 class ScrollIdForNode {
     private final String node;
-    private final SearchContextId contextId;
+    private final long scrollId;
     private final String clusterAlias;
 
-    ScrollIdForNode(@Nullable String clusterAlias, String node, SearchContextId contextId) {
+    ScrollIdForNode(@Nullable String clusterAlias, String node, long scrollId) {
         this.node = node;
         this.clusterAlias = clusterAlias;
-        this.contextId = contextId;
+        this.scrollId = scrollId;
     }
 
     public String getNode() {
@@ -42,15 +41,15 @@ public String getClusterAlias() {
         return clusterAlias;
     }
 
-    public SearchContextId getContextId() {
-        return contextId;
+    public long getScrollId() {
+        return scrollId;
     }
 
     @Override
     public String toString() {
         return "ScrollIdForNode{" +
             "node='" + node + '\'' +
-            ", scrollId=" + contextId +
+            ", scrollId=" + scrollId +
             ", clusterAlias='" + clusterAlias + '\'' +
             '}';
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 0eecfce9e1e56..0782fbb310b65 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -21,10 +21,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.transport.Transport;
@@ -39,23 +37,17 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction
     private final SearchPhaseController searchPhaseController;
 
     SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService,
-                                       final BiFunction<String, String, Transport.Connection> nodeIdToConnection,
-                                       final Map<String, AliasFilter> aliasFilter,
-                                       final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
-                                       final SearchPhaseController searchPhaseController, final Executor executor,
-                                       final SearchRequest request, final ActionListener<SearchResponse> listener,
-                                       final GroupShardsIterator<SearchShardIterator> shardsIts,
-                                       final TransportSearchAction.SearchTimeProvider timeProvider,
-                                       final ClusterState clusterState, final SearchTask task, SearchResponse.Clusters clusters) {
+            final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter,
+            final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
+            final SearchPhaseController searchPhaseController, final Executor executor,
+            final SearchRequest request, final ActionListener<SearchResponse> listener,
+            final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
+            final long clusterStateVersion, final SearchTask task, SearchResponse.Clusters clusters) {
         super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
                 executor, request, listener,
-                shardsIts, timeProvider, clusterState, task, new ArraySearchPhaseResults<>(shardsIts.size()),
+                shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()),
                 request.getMaxConcurrentShardRequests(), clusters);
         this.searchPhaseController = searchPhaseController;
-        SearchProgressListener progressListener = task.getProgressListener();
-        SearchSourceBuilder sourceBuilder = request.source();
-        progressListener.notifyListShards(SearchProgressListener.buildSearchShards(this.shardsIts),
-            SearchProgressListener.buildSearchShards(toSkipShardsIts), clusters, sourceBuilder == null || sourceBuilder.size() != 0);
     }
 
     @Override
@@ -68,6 +60,6 @@ protected void executePhaseOnShard(final SearchShardIterator shardIt, final Shar
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<DfsSearchResult> results, final SearchPhaseContext context) {
         return new DfsQueryPhase(results.getAtomicArray(), searchPhaseController, (queryResults) ->
-            new FetchSearchPhase(queryResults, searchPhaseController, context, clusterState()), context);
+            new FetchSearchPhase(queryResults, searchPhaseController, context), context);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
index e22104b8f70af..994ef5553bbc5 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.transport.Transport;
 
@@ -97,11 +96,11 @@ interface SearchPhaseContext extends Executor {
 
     /**
      * Releases a search context with the given context ID on the node the given connection is connected to.
-     * @see org.elasticsearch.search.query.QuerySearchResult#getContextId()
-     * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId()
+     * @see org.elasticsearch.search.query.QuerySearchResult#getRequestId()
+     * @see org.elasticsearch.search.fetch.FetchSearchResult#getRequestId()
      *
      */
-    default void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+    default void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
         if (connection != null) {
             getSearchTransport().sendFreeContext(connection, contextId, originalIndices);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 423f930b78fbe..59a5082ffe922 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -664,9 +664,9 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
                     }
                     numReducePhases++;
                     index = 1;
-                    if (hasAggs || hasTopDocs) {
-                        progressListener.notifyPartialReduce(SearchProgressListener.buildSearchShards(processedShards),
-                            topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0] : null, numReducePhases);
+                    if (hasAggs) {
+                        progressListener.notifyPartialReduce(progressListener.searchShards(processedShards),
+                            topDocsStats.getTotalHits(), aggsBuffer[0], numReducePhases);
                     }
                 }
                 final int i = index++;
@@ -695,8 +695,8 @@ private synchronized List<TopDocs> getRemainingTopDocs() {
         public ReducedQueryPhase reduce() {
             ReducedQueryPhase reducePhase = controller.reducedQueryPhase(results.asList(),
                 getRemainingAggs(), getRemainingTopDocs(), topDocsStats, numReducePhases, false, performFinalReduce);
-            progressListener.notifyFinalReduce(SearchProgressListener.buildSearchShards(results.asList()),
-                reducePhase.totalHits, reducePhase.aggregations, reducePhase.numReducePhases);
+            progressListener.notifyReduce(progressListener.searchShards(results.asList()),
+                reducePhase.totalHits, reducePhase.aggregations);
             return reducePhase;
         }
 
@@ -751,8 +751,7 @@ ReducedQueryPhase reduce() {
                 List<SearchPhaseResult> resultList = results.asList();
                 final ReducedQueryPhase reducePhase =
                     reducedQueryPhase(resultList, isScrollRequest, trackTotalHitsUpTo, request.isFinalReduce());
-                listener.notifyFinalReduce(SearchProgressListener.buildSearchShards(resultList),
-                    reducePhase.totalHits, reducePhase.aggregations, reducePhase.numReducePhases);
+                listener.notifyReduce(listener.searchShards(resultList), reducePhase.totalHits, reducePhase.aggregations);
                 return reducePhase;
             }
         };
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
index 997151160f96b..87146719a0f52 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
@@ -23,7 +23,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.action.search.SearchResponse.Clusters;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
@@ -49,27 +48,24 @@ abstract class SearchProgressListener {
      * Executed when shards are ready to be queried.
      *
      * @param shards The list of shards to query.
-     * @param skippedShards The list of skipped shards.
-     * @param clusters The statistics for remote clusters included in the search.
      * @param fetchPhase <code>true</code> if the search needs a fetch phase, <code>false</code> otherwise.
      **/
-    protected void onListShards(List<SearchShard> shards, List<SearchShard> skippedShards, Clusters clusters, boolean fetchPhase) {}
+    public void onListShards(List<SearchShard> shards, boolean fetchPhase) {}
 
     /**
      * Executed when a shard returns a query result.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards} )}.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)} )}.
      */
-    protected void onQueryResult(int shardIndex) {}
+    public void onQueryResult(int shardIndex) {}
 
     /**
      * Executed when a shard reports a query failure.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
-     * @param shardTarget The last shard target that thrown an exception.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}.
      * @param exc The cause of the failure.
      */
-    protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {}
+    public void onQueryFailure(int shardIndex, Exception exc) {}
 
     /**
      * Executed when a partial reduce is created. The number of partial reduce can be controlled via
@@ -78,9 +74,9 @@ protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exc
      * @param shards The list of shards that are part of this reduce.
      * @param totalHits The total number of hits in this reduce.
      * @param aggs The partial result for aggregations.
-     * @param reducePhase The version number for this reduce.
+     * @param version The version number for this reduce.
      */
-    protected void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {}
+    public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {}
 
     /**
      * Executed once when the final reduce is created.
@@ -88,29 +84,28 @@ protected void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, In
      * @param shards The list of shards that are part of this reduce.
      * @param totalHits The total number of hits in this reduce.
      * @param aggs The final result for aggregations.
-     * @param reducePhase The version number for this reduce.
      */
-    protected void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {}
+    public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {}
 
     /**
      * Executed when a shard returns a fetch result.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}.
      */
-    protected void onFetchResult(int shardIndex) {}
+    public void onFetchResult(int shardIndex) {}
 
     /**
      * Executed when a shard reports a fetch failure.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}.
      * @param exc The cause of the failure.
      */
-    protected void onFetchFailure(int shardIndex, Exception exc) {}
+    public void onFetchFailure(int shardIndex, Exception exc) {}
 
-    final void notifyListShards(List<SearchShard> shards, List<SearchShard> skippedShards, Clusters clusters, boolean fetchPhase) {
+    final void notifyListShards(List<SearchShard> shards, boolean fetchPhase) {
         this.shards = shards;
         try {
-            onListShards(shards, skippedShards, clusters, fetchPhase);
+            onListShards(shards, fetchPhase);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on list shards"), e);
         }
@@ -125,26 +120,26 @@ final void notifyQueryResult(int shardIndex) {
         }
     }
 
-    final void notifyQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
+    final void notifyQueryFailure(int shardIndex, Exception exc) {
         try {
-            onQueryFailure(shardIndex, shardTarget, exc);
+            onQueryFailure(shardIndex, exc);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("[{}] Failed to execute progress listener on query failure",
                 shards.get(shardIndex)), e);
         }
     }
 
-    final void notifyPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+    final void notifyPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {
         try {
-            onPartialReduce(shards, totalHits, aggs, reducePhase);
+            onPartialReduce(shards, totalHits, aggs, version);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on partial reduce"), e);
         }
     }
 
-    protected final void notifyFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+    final void notifyReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {
         try {
-            onFinalReduce(shards, totalHits, aggs, reducePhase);
+            onReduce(shards, totalHits, aggs);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on reduce"), e);
         }
@@ -168,7 +163,7 @@ final void notifyFetchFailure(int shardIndex, Exception exc) {
         }
     }
 
-    static List<SearchShard> buildSearchShards(List<? extends SearchPhaseResult> results) {
+    final List<SearchShard> searchShards(List<? extends SearchPhaseResult> results) {
         return results.stream()
             .filter(Objects::nonNull)
             .map(SearchPhaseResult::getSearchShardTarget)
@@ -176,14 +171,14 @@ static List<SearchShard> buildSearchShards(List<? extends SearchPhaseResult> res
             .collect(Collectors.toUnmodifiableList());
     }
 
-    static List<SearchShard> buildSearchShards(SearchShardTarget[] results) {
+    final List<SearchShard> searchShards(SearchShardTarget[] results) {
         return Arrays.stream(results)
             .filter(Objects::nonNull)
             .map(e -> new SearchShard(e.getClusterAlias(), e.getShardId()))
             .collect(Collectors.toUnmodifiableList());
     }
 
-    static List<SearchShard> buildSearchShards(GroupShardsIterator<SearchShardIterator> its) {
+    final List<SearchShard> searchShards(GroupShardsIterator<SearchShardIterator> its) {
         return StreamSupport.stream(its.spliterator(), false)
             .map(e -> new SearchShard(e.getClusterAlias(), e.shardId()))
             .collect(Collectors.toUnmodifiableList());
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index 24345c606003a..d5060b728347d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -21,11 +21,9 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.search.SearchPhaseResult;
-import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.transport.Transport;
@@ -41,24 +39,22 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<Se
     private final SearchProgressListener progressListener;
 
     SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService,
-                                    final BiFunction<String, String, Transport.Connection> nodeIdToConnection,
-                                    final Map<String, AliasFilter> aliasFilter,
-                                    final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
-                                    final SearchPhaseController searchPhaseController, final Executor executor,
-                                    final SearchRequest request, final ActionListener<SearchResponse> listener,
-                                    final GroupShardsIterator<SearchShardIterator> shardsIts,
-                                    final TransportSearchAction.SearchTimeProvider timeProvider,
-                                    ClusterState clusterState, SearchTask task, SearchResponse.Clusters clusters) {
+            final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter,
+            final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
+            final SearchPhaseController searchPhaseController, final Executor executor,
+            final SearchRequest request, final ActionListener<SearchResponse> listener,
+            final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
+            long clusterStateVersion, SearchTask task, SearchResponse.Clusters clusters) {
         super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
-                executor, request, listener, shardsIts, timeProvider, clusterState, task,
+                executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task,
                 searchPhaseController.newSearchPhaseResults(task.getProgressListener(), request, shardsIts.size()),
                 request.getMaxConcurrentShardRequests(), clusters);
         this.searchPhaseController = searchPhaseController;
         this.progressListener = task.getProgressListener();
         final SearchProgressListener progressListener = task.getProgressListener();
         final SearchSourceBuilder sourceBuilder = request.source();
-        progressListener.notifyListShards(SearchProgressListener.buildSearchShards(this.shardsIts),
-            SearchProgressListener.buildSearchShards(toSkipShardsIts), clusters, sourceBuilder == null || sourceBuilder.size() != 0);
+        progressListener.notifyListShards(progressListener.searchShards(this.shardsIts),
+            sourceBuilder == null || sourceBuilder.size() != 0);
     }
 
     protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard,
@@ -68,12 +64,12 @@ protected void executePhaseOnShard(final SearchShardIterator shardIt, final Shar
     }
 
     @Override
-    protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
-        progressListener.notifyQueryFailure(shardIndex, shardTarget, exc);
+    protected void onShardGroupFailure(int shardIndex, Exception exc) {
+        progressListener.notifyQueryFailure(shardIndex, exc);
     }
 
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, final SearchPhaseContext context) {
-        return new FetchSearchPhase(results, searchPhaseController, context, clusterState());
+        return new FetchSearchPhase(results, searchPhaseController, context);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 96206aa4bcd58..8ba9a8c9f0bd7 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
@@ -55,9 +56,9 @@
  * @see org.elasticsearch.client.Client#search(SearchRequest)
  * @see SearchResponse
  */
-public class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable {
+public final class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable {
 
-    public static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
+    private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
 
     public static final int DEFAULT_PRE_FILTER_SHARD_SIZE = 128;
     public static final int DEFAULT_BATCHED_REDUCE_SIZE = 512;
@@ -559,7 +560,7 @@ public boolean isSuggestOnly() {
     }
 
     @Override
-    public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
         // generating description in a lazy way since source can be quite big
         return new SearchTask(id, type, action, null, parentTaskId, headers) {
             @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index b8b791360d30c..ceaee96f5c131 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -213,7 +213,7 @@ public SearchRequestBuilder setVersion(boolean version) {
         sourceBuilder().version(version);
         return this;
     }
-
+    
     /**
      * Should each {@link org.elasticsearch.search.SearchHit} be returned with the
      * sequence number and primary term of the last modification of the document.
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 81d61f2996ef4..cb36dbd0cd8f2 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -113,10 +113,6 @@ public RestStatus status() {
         return RestStatus.status(successfulShards, totalShards, shardFailures);
     }
 
-    public SearchResponseSections getInternalResponse() {
-        return internalResponse;
-    }
-
     /**
      * The search hits.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
index edc3f1b96a143..46de6de8f87c9 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.Transport;
 
@@ -148,11 +147,11 @@ private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, fi
                 }
                 connection = getConnection(target.getClusterAlias(), node);
             } catch (Exception ex) {
-                onShardFailure("query", counter, target.getContextId(),
+                onShardFailure("query", counter, target.getScrollId(),
                     ex, null, () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
                 continue;
             }
-            final InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(target.getContextId(), request);
+            final InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(target.getScrollId(), request);
             // we can't create a SearchShardTarget here since we don't know the index and shard ID we are talking to
             // we only know the node and the search context ID. Yet, the response will contain the SearchShardTarget
             // from the target node instead...that's why we pass null here
@@ -192,7 +191,7 @@ protected void innerOnResponse(T result) {
 
                 @Override
                 public void onFailure(Exception t) {
-                    onShardFailure("query", counter, target.getContextId(), t, null,
+                    onShardFailure("query", counter, target.getScrollId(), t, null,
                         () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
                 }
             };
@@ -248,7 +247,7 @@ protected final void sendResponse(SearchPhaseController.ReducedQueryPhase queryP
         }
     }
 
-    protected void onShardFailure(String phaseName, final CountDown counter, final SearchContextId searchId, Exception failure,
+    protected void onShardFailure(String phaseName, final CountDown counter, final long searchId, Exception failure,
                                   @Nullable SearchShardTarget searchShardTarget,
                                   Supplier<SearchPhase> nextPhaseSupplier) {
         if (logger.isDebugEnabled()) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index d87ef021bfa59..df18296de2a4a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -86,7 +86,7 @@ public void run() {
                     if (docIds != null) {
                         final QuerySearchResult querySearchResult = queryResults.get(index);
                         ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[index];
-                        ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.getContextId(), docIds,
+                        ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.getRequestId(), docIds,
                             lastEmittedDoc);
                         SearchShardTarget searchShardTarget = querySearchResult.getSearchShardTarget();
                         DiscoveryNode node = clusterNodeLookup.apply(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
@@ -104,7 +104,7 @@ protected void innerOnResponse(FetchSearchResult response) {
 
                                 @Override
                                 public void onFailure(Exception t) {
-                                    onShardFailure(getName(), counter, querySearchResult.getContextId(),
+                                    onShardFailure(getName(), counter, querySearchResult.getRequestId(),
                                         t, querySearchResult.getSearchShardTarget(),
                                         () -> sendResponsePhase(reducedQueryPhase, fetchResults));
                                 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java
index 8a57ce51a7466..16459d81885ce 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java
@@ -29,18 +29,19 @@
  * A class that encapsulates the {@link ShardId} and the cluster alias
  * of a shard used during the search action.
  */
-public final class SearchShard implements Comparable<SearchShard> {
+public class SearchShard implements Comparable<SearchShard> {
     @Nullable
     private final String clusterAlias;
     private final ShardId shardId;
 
-    public SearchShard(@Nullable String clusterAlias, ShardId shardId) {
+    SearchShard(@Nullable String clusterAlias, ShardId shardId) {
         this.clusterAlias = clusterAlias;
         this.shardId = shardId;
     }
 
     /**
-     * Return the cluster alias if we are executing a cross cluster search request, <code>null</code> otherwise.
+     * Return the cluster alias if the shard is on a remote cluster and <code>null</code>
+     * otherwise (local).
      */
     @Nullable
     public String getClusterAlias() {
@@ -50,6 +51,7 @@ public String getClusterAlias() {
     /**
      * Return the {@link ShardId} of this shard.
      */
+    @Nullable
     public ShardId getShardId() {
         return shardId;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java
index abfc876ad6000..4719c1fda9d53 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java
@@ -40,4 +40,5 @@ public SearchShardTask(long id, String type, String action, String description,
     public boolean shouldCancelChildrenOnCancellation() {
         return false;
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
index c5a918c06f1bb..97247e443bb64 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
@@ -37,14 +37,14 @@ public SearchTask(long id, String type, String action, String description, TaskI
     /**
      * Attach a {@link SearchProgressListener} to this task.
      */
-    public final void setProgressListener(SearchProgressListener progressListener) {
+    public void setProgressListener(SearchProgressListener progressListener) {
         this.progressListener = progressListener;
     }
 
     /**
      * Return the {@link SearchProgressListener} attached to this task.
      */
-    public final SearchProgressListener getProgressListener() {
+    public SearchProgressListener getProgressListener() {
         return progressListener;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 681177e4399cd..16e8c17688906 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -88,7 +87,7 @@ public SearchTransportService(TransportService transportService,
         this.responseWrapper = responseWrapper;
     }
 
-    public void sendFreeContext(Transport.Connection connection, final SearchContextId contextId, OriginalIndices originalIndices) {
+    public void sendFreeContext(Transport.Connection connection, final long contextId, OriginalIndices originalIndices) {
         transportService.sendRequest(connection, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(originalIndices, contextId),
             TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(new ActionListener<SearchFreeContextResponse>() {
                 @Override
@@ -103,8 +102,7 @@ public void onFailure(Exception e) {
             }, SearchFreeContextResponse::new));
     }
 
-    public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
-                                ActionListener<SearchFreeContextResponse> listener) {
+    public void sendFreeContext(Transport.Connection connection, long contextId, final ActionListener<SearchFreeContextResponse> listener) {
         transportService.sendRequest(connection, FREE_CONTEXT_SCROLL_ACTION_NAME, new ScrollFreeContextRequest(contextId),
             TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(listener, SearchFreeContextResponse::new));
     }
@@ -196,33 +194,39 @@ public Map<String, Long> getPendingSearchRequests() {
     }
 
     static class ScrollFreeContextRequest extends TransportRequest {
-        private SearchContextId contextId;
+        private long id;
 
-        ScrollFreeContextRequest(SearchContextId contextId) {
-            this.contextId = contextId;
+        ScrollFreeContextRequest() {
+        }
+
+        ScrollFreeContextRequest(long id) {
+            this.id = id;
         }
 
         ScrollFreeContextRequest(StreamInput in) throws IOException {
             super(in);
-            contextId = new SearchContextId(in);
+            id = in.readLong();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            contextId.writeTo(out);
+            out.writeLong(id);
         }
 
-        public SearchContextId id() {
-            return this.contextId;
+        public long id() {
+            return this.id;
         }
 
-    }
+        }
 
     static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest {
         private OriginalIndices originalIndices;
 
-        SearchFreeContextRequest(OriginalIndices originalIndices, SearchContextId id) {
+        SearchFreeContextRequest() {
+        }
+
+        SearchFreeContextRequest(OriginalIndices originalIndices, long id) {
             super(id);
             this.originalIndices = originalIndices;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index d7f60de9cffc0..2f7e8e338ea24 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -511,7 +511,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         BiFunction<String, String, Transport.Connection> connectionLookup = buildConnectionLookup(searchRequest.getLocalClusterAlias(),
             nodes::get, remoteConnections, searchTransportService::getConnection);
         boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators);
-        searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState,
+        searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState.version(),
             Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, routingMap, listener, preFilterSearchShards, clusters).start();
     }
 
@@ -560,7 +560,7 @@ private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction
                                                         GroupShardsIterator<SearchShardIterator> shardIterators,
                                                         SearchTimeProvider timeProvider,
                                                         BiFunction<String, String, Transport.Connection> connectionLookup,
-                                                        ClusterState clusterState,
+                                                        long clusterStateVersion,
                                                         Map<String, AliasFilter> aliasFilter,
                                                         Map<String, Float> concreteIndexBoosts,
                                                         Map<String, Set<String>> indexRoutings,
@@ -571,14 +571,14 @@ private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction
         if (preFilter) {
             return new CanMatchPreFilterSearchPhase(logger, searchTransportService, connectionLookup,
                 aliasFilter, concreteIndexBoosts, indexRoutings, executor, searchRequest, listener, shardIterators,
-                timeProvider, clusterState, task, (iter) -> {
+                timeProvider, clusterStateVersion, task, (iter) -> {
                 AbstractSearchAsyncAction<? extends SearchPhaseResult> action = searchAsyncAction(
                     task,
                     searchRequest,
                     iter,
                     timeProvider,
                     connectionLookup,
-                    clusterState,
+                    clusterStateVersion,
                     aliasFilter,
                     concreteIndexBoosts,
                     indexRoutings,
@@ -598,12 +598,12 @@ public void run() {
                 case DFS_QUERY_THEN_FETCH:
                     searchAsyncAction = new SearchDfsQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
                         aliasFilter, concreteIndexBoosts, indexRoutings, searchPhaseController, executor, searchRequest, listener,
-                        shardIterators, timeProvider, clusterState, task, clusters);
+                        shardIterators, timeProvider, clusterStateVersion, task, clusters);
                     break;
                 case QUERY_THEN_FETCH:
                     searchAsyncAction = new SearchQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
                         aliasFilter, concreteIndexBoosts, indexRoutings, searchPhaseController, executor, searchRequest, listener,
-                        shardIterators, timeProvider, clusterState, task, clusters);
+                        shardIterators, timeProvider, clusterStateVersion, task, clusters);
                     break;
                 default:
                     throw new IllegalStateException("Unknown search type: [" + searchRequest.searchType() + "]");
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index f3755180b1e62..c848e227af4c0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
@@ -33,25 +32,16 @@
 
 final class TransportSearchHelper {
 
-    private static final String INCLUDE_CONTEXT_UUID = "include_context_uuid";
-
-    static InternalScrollSearchRequest internalScrollSearchRequest(SearchContextId id, SearchScrollRequest request) {
+    static InternalScrollSearchRequest internalScrollSearchRequest(long id, SearchScrollRequest request) {
         return new InternalScrollSearchRequest(request, id);
     }
 
-    static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults,
-                                boolean includeContextUUID) throws IOException {
+    static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
         try (RAMOutputStream out = new RAMOutputStream()) {
-            if (includeContextUUID) {
-                out.writeString(INCLUDE_CONTEXT_UUID);
-            }
             out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE);
             out.writeVInt(searchPhaseResults.asList().size());
             for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) {
-                if (includeContextUUID) {
-                    out.writeString(searchPhaseResult.getContextId().getReaderId());
-                }
-                out.writeLong(searchPhaseResult.getContextId().getId());
+                out.writeLong(searchPhaseResult.getRequestId());
                 SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget();
                 if (searchShardTarget.getClusterAlias() != null) {
                     out.writeString(
@@ -70,19 +60,9 @@ static ParsedScrollId parseScrollId(String scrollId) {
         try {
             byte[] bytes = Base64.getUrlDecoder().decode(scrollId);
             ByteArrayDataInput in = new ByteArrayDataInput(bytes);
-            final boolean includeContextUUID;
-            final String type;
-            final String firstChunk = in.readString();
-            if (INCLUDE_CONTEXT_UUID.equals(firstChunk)) {
-                includeContextUUID = true;
-                type = in.readString();
-            } else {
-                includeContextUUID = false;
-                type = firstChunk;
-            }
+            String type = in.readString();
             ScrollIdForNode[] context = new ScrollIdForNode[in.readVInt()];
             for (int i = 0; i < context.length; ++i) {
-                final String contextUUID = includeContextUUID ? in.readString() : "";
                 long id = in.readLong();
                 String target = in.readString();
                 String clusterAlias;
@@ -93,7 +73,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
                     clusterAlias = target.substring(0, index);
                     target = target.substring(index+1);
                 }
-                context[i] = new ScrollIdForNode(clusterAlias, target, new SearchContextId(contextUUID, id));
+                context[i] = new ScrollIdForNode(clusterAlias, target, id);
             }
             if (in.getPosition() != bytes.length) {
                 throw new IllegalArgumentException("Not all bytes were read");
diff --git a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
index cf4ab92baa0c6..40bbf81534b58 100644
--- a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
+++ b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
@@ -23,6 +23,12 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchTask;
+import org.elasticsearch.action.search.SearchProgressActionListener;
+import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.support.AbstractClient;
@@ -102,6 +108,38 @@ > Task executeLocally(ActionType<Response> action, Request request, TaskListener
             listener::onResponse, listener::onFailure);
     }
 
+    /**
+     * Execute a {@link SearchRequest} locally and track the progress of the request through
+     * a {@link SearchProgressActionListener}.
+     */
+    public SearchTask executeSearchLocally(SearchRequest request, SearchProgressActionListener listener) {
+        // we cannot track the progress if remote cluster requests are splitted.
+        request.setCcsMinimizeRoundtrips(false);
+        TransportSearchAction action = (TransportSearchAction) actions.get(SearchAction.INSTANCE);
+        SearchTask task = (SearchTask) taskManager.register("transport", action.actionName, request);
+        task.setProgressListener(listener);
+        action.execute(task, request, new ActionListener<>() {
+            @Override
+            public void onResponse(SearchResponse response) {
+                try {
+                    taskManager.unregister(task);
+                } finally {
+                    listener.onResponse(response);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    taskManager.unregister(task);
+                } finally {
+                    listener.onFailure(e);
+                }
+            }
+        });
+        return task;
+    }
+
     /**
      * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
      * {@link #executeLocally(ActionType, ActionRequest, TaskListener)}.
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 70138ecc981bc..e445615e0fc73 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -128,8 +127,6 @@ public static List<Entry> getNamedWriteables() {
         registerMetaDataCustom(entries, IndexGraveyard.TYPE, IndexGraveyard::new, IndexGraveyard::readDiffFrom);
         registerMetaDataCustom(entries, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new,
             PersistentTasksCustomMetaData::readDiffFrom);
-        registerMetaDataCustom(entries, ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new,
-            ComponentTemplateMetadata::readDiffFrom);
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
         return entries;
@@ -148,8 +145,6 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
             IndexGraveyard::fromXContent));
         entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE),
             PersistentTasksCustomMetaData::fromXContent));
-        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(ComponentTemplateMetadata.TYPE),
-            ComponentTemplateMetadata::fromXContent));
         return entries;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index acd3294f38641..b004af9d38ab6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -304,32 +304,20 @@ private class ConnectionTarget {
             @Override
             protected void doRun() {
                 assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
-                if (transportService.nodeConnected(discoveryNode)) {
-                    // transportService.connectToNode is a no-op if already connected, but we don't want any DEBUG logging in this case
-                    // since we run this for every node on every cluster state update.
-                    logger.trace("still connected to {}", discoveryNode);
-                    onConnected();
-                } else {
-                    logger.debug("connecting to {}", discoveryNode);
-                    transportService.connectToNode(discoveryNode, new ActionListener<Void>() {
-                        @Override
-                        public void onResponse(Void aVoid) {
-                            assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
-                            logger.debug("connected to {}", discoveryNode);
-                            onConnected();
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            abstractRunnable.onFailure(e);
-                        }
-                    });
-                }
-            }
+                transportService.connectToNode(discoveryNode, new ActionListener<Void>() {
+                    @Override
+                    public void onResponse(Void aVoid) {
+                        assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
+                        consecutiveFailureCount.set(0);
+                        logger.debug("connected to {}", discoveryNode);
+                        onCompletion(ActivityType.CONNECTING, null, disconnectActivity);
+                    }
 
-            private void onConnected() {
-                consecutiveFailureCount.set(0);
-                onCompletion(ActivityType.CONNECTING, null, disconnectActivity);
+                    @Override
+                    public void onFailure(Exception e) {
+                        abstractRunnable.onFailure(e);
+                    }
+                });
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
index 479de5db45a18..16e75464caf0f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
@@ -495,8 +495,6 @@ public void onFailure(Exception e) {
     private void processJoinRequest(JoinRequest joinRequest, JoinHelper.JoinCallback joinCallback) {
         final Optional<Join> optionalJoin = joinRequest.getOptionalJoin();
         synchronized (mutex) {
-            updateMaxTermSeen(joinRequest.getTerm());
-
             final CoordinationState coordState = coordinationState.get();
             final boolean prevElectionWon = coordState.electionWon();
 
@@ -1117,7 +1115,7 @@ private class CoordinatorPeerFinder extends PeerFinder {
         protected void onActiveMasterFound(DiscoveryNode masterNode, long term) {
             synchronized (mutex) {
                 ensureTermAtLeast(masterNode, term);
-                joinHelper.sendJoinRequest(masterNode, getCurrentTerm(), joinWithDestination(lastJoin, masterNode, term));
+                joinHelper.sendJoinRequest(masterNode, joinWithDestination(lastJoin, masterNode, term));
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
index 03f7b37d2b835..2e47acf07b215 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
@@ -130,7 +130,7 @@ public ClusterTasksResult<JoinTaskExecutor.Task> execute(ClusterState currentSta
             StartJoinRequest::new,
             (request, channel, task) -> {
                 final DiscoveryNode destination = request.getSourceNode();
-                sendJoinRequest(destination, currentTermSupplier.getAsLong(), Optional.of(joinLeaderInTerm.apply(request)));
+                sendJoinRequest(destination, Optional.of(joinLeaderInTerm.apply(request)));
                 channel.sendResponse(Empty.INSTANCE);
             });
 
@@ -230,9 +230,9 @@ void logLastFailedJoinAttempt() {
         }
     }
 
-    public void sendJoinRequest(DiscoveryNode destination, long term, Optional<Join> optionalJoin) {
+    public void sendJoinRequest(DiscoveryNode destination, Optional<Join> optionalJoin) {
         assert destination.isMasterNode() : "trying to join master-ineligible " + destination;
-        final JoinRequest joinRequest = new JoinRequest(transportService.getLocalNode(), term, optionalJoin);
+        final JoinRequest joinRequest = new JoinRequest(transportService.getLocalNode(), optionalJoin);
         final Tuple<DiscoveryNode, JoinRequest> dedupKey = Tuple.tuple(destination, joinRequest);
         if (pendingOutgoingJoins.add(dedupKey)) {
             logger.debug("attempting to join {} with {}", destination, joinRequest);
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java
index fe1ce7d23ca44..091a6809c84dc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java
@@ -18,53 +18,29 @@
  */
 package org.elasticsearch.cluster.coordination;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
-import java.util.Objects;
 import java.util.Optional;
 
 public class JoinRequest extends TransportRequest {
 
-    /**
-     * The sending (i.e. joining) node.
-     */
     private final DiscoveryNode sourceNode;
 
-    /**
-     * The minimum term for which the joining node will accept any cluster state publications. If the joining node is in a strictly greater
-     * term than the master it wants to join then the master must enter a new term and hold another election. Doesn't necessarily match
-     * {@link JoinRequest#optionalJoin}.
-     */
-    private final long minimumTerm;
-
-    /**
-     * A vote for the receiving node. This vote is optional since the sending node may have voted for a different master in this term.
-     * That's ok, the sender likely discovered that the master we voted for lost the election and now we're trying to join the winner. Once
-     * the sender has successfully joined the master, the lack of a vote in its term causes another election (see
-     * {@link Publication#onMissingJoin(DiscoveryNode)}).
-     */
     private final Optional<Join> optionalJoin;
 
-    public JoinRequest(DiscoveryNode sourceNode, long minimumTerm, Optional<Join> optionalJoin) {
+    public JoinRequest(DiscoveryNode sourceNode, Optional<Join> optionalJoin) {
         assert optionalJoin.isPresent() == false || optionalJoin.get().getSourceNode().equals(sourceNode);
         this.sourceNode = sourceNode;
-        this.minimumTerm = minimumTerm;
         this.optionalJoin = optionalJoin;
     }
 
     public JoinRequest(StreamInput in) throws IOException {
         super(in);
         sourceNode = new DiscoveryNode(in);
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-            minimumTerm = in.readLong();
-        } else {
-            minimumTerm = 0L;
-        }
         optionalJoin = Optional.ofNullable(in.readOptionalWriteable(Join::new));
     }
 
@@ -72,9 +48,6 @@ public JoinRequest(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         sourceNode.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-            out.writeLong(minimumTerm);
-        }
         out.writeOptionalWriteable(optionalJoin.orElse(null));
     }
 
@@ -82,17 +55,6 @@ public DiscoveryNode getSourceNode() {
         return sourceNode;
     }
 
-    public long getMinimumTerm() {
-        return minimumTerm;
-    }
-
-    public long getTerm() {
-        // If the join is also present then its term will normally equal the corresponding term, but we do not require callers to
-        // obtain the term and the join in a synchronized fashion so it's possible that they disagree. Also older nodes do not share the
-        // minimum term, so for BWC we can take it from the join if present.
-        return Math.max(minimumTerm, optionalJoin.map(Join::getTerm).orElse(0L));
-    }
-
     public Optional<Join> getOptionalJoin() {
         return optionalJoin;
     }
@@ -104,21 +66,21 @@ public boolean equals(Object o) {
 
         JoinRequest that = (JoinRequest) o;
 
-        if (minimumTerm != that.minimumTerm) return false;
         if (!sourceNode.equals(that.sourceNode)) return false;
         return optionalJoin.equals(that.optionalJoin);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(sourceNode, minimumTerm, optionalJoin);
+        int result = sourceNode.hashCode();
+        result = 31 * result + optionalJoin.hashCode();
+        return result;
     }
 
     @Override
     public String toString() {
         return "JoinRequest{" +
             "sourceNode=" + sourceNode +
-            ", minimumTerm=" + minimumTerm +
             ", optionalJoin=" + optionalJoin +
             '}';
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
deleted file mode 100644
index 2b160b1b2fb11..0000000000000
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.cluster.AbstractDiffable;
-import org.elasticsearch.cluster.Diff;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * A component template is a re-usable template as well as metadata about the template. Each
- * component template is expected to be valid on its own. For example, if a component template
- * contains a field "foo", it's expected to contain all the necessary settings/mappings/etc for the
- * "foo" field. These component templates make up the individual pieces composing an index template.
- */
-public class ComponentTemplate extends AbstractDiffable<ComponentTemplate> implements ToXContentObject {
-    private static final ParseField TEMPLATE = new ParseField("template");
-    private static final ParseField VERSION = new ParseField("version");
-    private static final ParseField METADATA = new ParseField("_meta");
-
-    @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<ComponentTemplate, Void> PARSER =
-        new ConstructingObjectParser<>("component_template", false,
-            a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map<String, Object>) a[2]));
-
-    static {
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Template.PARSER, TEMPLATE);
-        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION);
-        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
-    }
-
-    private final Template template;
-    @Nullable
-    private final Long version;
-    @Nullable
-    private final Map<String, Object> metadata;
-
-    static Diff<ComponentTemplate> readComponentTemplateDiffFrom(StreamInput in) throws IOException {
-        return AbstractDiffable.readDiffFrom(ComponentTemplate::new, in);
-    }
-
-    public static ComponentTemplate parse(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
-    public ComponentTemplate(Template template, @Nullable Long version, @Nullable Map<String, Object> metadata) {
-        this.template = template;
-        this.version = version;
-        this.metadata = metadata;
-    }
-
-    public ComponentTemplate(StreamInput in) throws IOException {
-        this.template = new Template(in);
-        this.version = in.readOptionalVLong();
-        if (in.readBoolean()) {
-            this.metadata = in.readMap();
-        } else {
-            this.metadata = null;
-        }
-    }
-
-    public Template template() {
-        return template;
-    }
-
-    @Nullable
-    public Long version() {
-        return version;
-    }
-
-    @Nullable
-    public Map<String, Object> metadata() {
-        return metadata;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        this.template.writeTo(out);
-        out.writeOptionalVLong(this.version);
-        if (this.metadata == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeMap(this.metadata);
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(template, version, metadata);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (obj == null) {
-            return false;
-        }
-        if (obj.getClass() != getClass()) {
-            return false;
-        }
-        ComponentTemplate other = (ComponentTemplate) obj;
-        return Objects.equals(template, other.template) &&
-            Objects.equals(version, other.version) &&
-            Objects.equals(metadata, other.metadata);
-    }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field(TEMPLATE.getPreferredName(), this.template);
-        if (this.version != null) {
-            builder.field(VERSION.getPreferredName(), this.version);
-        }
-        if (this.metadata != null) {
-            builder.field(METADATA.getPreferredName(), this.metadata);
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    static class Template extends AbstractDiffable<Template> implements ToXContentObject {
-        private static final ParseField SETTINGS = new ParseField("settings");
-        private static final ParseField MAPPINGS = new ParseField("mappings");
-        private static final ParseField ALIASES = new ParseField("aliases");
-
-        @SuppressWarnings("unchecked")
-        private static final ConstructingObjectParser<Template, Void> PARSER = new ConstructingObjectParser<>("template", false,
-            a -> new Template((Settings) a[0], (CompressedXContent) a[1], (Map<String, AliasMetaData>) a[2]));
-
-        static {
-            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
-            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
-                new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered()))), MAPPINGS);
-            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
-                Map<String, AliasMetaData> aliasMap = new HashMap<>();
-                while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    AliasMetaData alias = AliasMetaData.Builder.fromXContent(p);
-                    aliasMap.put(alias.alias(), alias);
-                }
-                return aliasMap;
-            }, ALIASES);
-        }
-
-        @Nullable
-        private final Settings settings;
-        @Nullable
-        private final CompressedXContent mappings;
-        @Nullable
-        private final Map<String, AliasMetaData> aliases;
-
-        Template(@Nullable Settings settings, @Nullable CompressedXContent mappings, @Nullable Map<String, AliasMetaData> aliases) {
-            this.settings = settings;
-            this.mappings = mappings;
-            this.aliases = aliases;
-        }
-
-        Template(StreamInput in) throws IOException {
-            if (in.readBoolean()) {
-                this.settings = Settings.readSettingsFromStream(in);
-            } else {
-                this.settings = null;
-            }
-            if (in.readBoolean()) {
-                this.mappings = CompressedXContent.readCompressedString(in);
-            } else {
-                this.mappings = null;
-            }
-            if (in.readBoolean()) {
-                this.aliases = in.readMap(StreamInput::readString, AliasMetaData::new);
-            } else {
-                this.aliases = null;
-            }
-        }
-
-        public Settings settings() {
-            return settings;
-        }
-
-        public CompressedXContent mappings() {
-            return mappings;
-        }
-
-        public Map<String, AliasMetaData> aliases() {
-            return aliases;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            if (this.settings == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                Settings.writeSettingsToStream(this.settings, out);
-            }
-            if (this.mappings == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                this.mappings.writeTo(out);
-            }
-            if (this.aliases == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                out.writeMap(this.aliases, StreamOutput::writeString, (stream, aliasMetaData) -> aliasMetaData.writeTo(stream));
-            }
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(settings, mappings, aliases);
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (obj == null) {
-                return false;
-            }
-            if (obj.getClass() != getClass()) {
-                return false;
-            }
-            Template other = (Template) obj;
-            return Objects.equals(settings, other.settings) &&
-                Objects.equals(mappings, other.mappings) &&
-                Objects.equals(aliases, other.aliases);
-        }
-
-        @Override
-        public String toString() {
-            return Strings.toString(this);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            if (this.settings != null) {
-                builder.startObject(SETTINGS.getPreferredName());
-                this.settings.toXContent(builder, params);
-                builder.endObject();
-            }
-            if (this.mappings != null) {
-                Map<String, Object> uncompressedMapping =
-                    XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2();
-                if (uncompressedMapping.size() > 0) {
-                    builder.field(MAPPINGS.getPreferredName());
-                    builder.map(uncompressedMapping);
-                }
-            }
-            if (this.aliases != null) {
-                builder.startObject(ALIASES.getPreferredName());
-                for (AliasMetaData alias : this.aliases.values()) {
-                    AliasMetaData.Builder.toXContent(alias, builder, params);
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            return builder;
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
deleted file mode 100644
index 1b1e6b346ef9a..0000000000000
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.Diff;
-import org.elasticsearch.cluster.DiffableUtils;
-import org.elasticsearch.cluster.NamedDiff;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * {@link ComponentTemplateMetadata} is a custom {@link MetaData} implementation for storing a map
- * of component templates and their names.
- */
-public class ComponentTemplateMetadata implements MetaData.Custom {
-    public static final String TYPE = "component_template";
-    private static final ParseField COMPONENT_TEMPLATE = new ParseField("component_template");
-    @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<ComponentTemplateMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE, false,
-        a -> new ComponentTemplateMetadata((Map<String, ComponentTemplate>) a[0]));
-
-    static {
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
-            Map<String, ComponentTemplate> templates = new HashMap<>();
-            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
-                String name = p.currentName();
-                templates.put(name, ComponentTemplate.parse(p));
-            }
-            return templates;
-        }, COMPONENT_TEMPLATE);
-    }
-    private final Map<String, ComponentTemplate> componentTemplates;
-
-    public ComponentTemplateMetadata(Map<String, ComponentTemplate> componentTemplates) {
-        this.componentTemplates = componentTemplates;
-    }
-
-    public ComponentTemplateMetadata(StreamInput in) throws IOException {
-        this.componentTemplates = in.readMap(StreamInput::readString, ComponentTemplate::new);
-    }
-
-    public Map<String, ComponentTemplate> componentTemplates() {
-        return this.componentTemplates;
-    }
-
-    @Override
-    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
-        return new ComponentTemplateMetadataDiff((ComponentTemplateMetadata) before, this);
-    }
-
-    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
-        return new ComponentTemplateMetadataDiff(in);
-    }
-
-    @Override
-    public EnumSet<MetaData.XContentContext> context() {
-        return MetaData.ALL_CONTEXTS;
-    }
-
-    @Override
-    public String getWriteableName() {
-        return TYPE;
-    }
-
-    @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeMap(this.componentTemplates, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
-    }
-
-    public static ComponentTemplateMetadata fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(COMPONENT_TEMPLATE.getPreferredName());
-        for (Map.Entry<String, ComponentTemplate> template : componentTemplates.entrySet()) {
-            builder.field(template.getKey(), template.getValue());
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(this.componentTemplates);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (obj == null) {
-            return false;
-        }
-        if (obj.getClass() != getClass()) {
-            return false;
-        }
-        ComponentTemplateMetadata other = (ComponentTemplateMetadata) obj;
-        return Objects.equals(this.componentTemplates, other.componentTemplates);
-    }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this);
-    }
-
-    static class ComponentTemplateMetadataDiff implements NamedDiff<MetaData.Custom> {
-
-        final Diff<Map<String, ComponentTemplate>> componentTemplateDiff;
-
-        ComponentTemplateMetadataDiff(ComponentTemplateMetadata before, ComponentTemplateMetadata after) {
-            this.componentTemplateDiff = DiffableUtils.diff(before.componentTemplates, after.componentTemplates,
-                DiffableUtils.getStringKeySerializer());
-        }
-
-        ComponentTemplateMetadataDiff(StreamInput in) throws IOException {
-            this.componentTemplateDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
-                ComponentTemplate::new, ComponentTemplate::readComponentTemplateDiffFrom);
-        }
-
-        @Override
-        public MetaData.Custom apply(MetaData.Custom part) {
-            return new ComponentTemplateMetadata(componentTemplateDiff.apply(((ComponentTemplateMetadata) part).componentTemplates));
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            componentTemplateDiff.writeTo(out);
-        }
-
-        @Override
-        public String getWriteableName() {
-            return TYPE;
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 6154e6ecd675f..00d3a8d708d62 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -73,7 +73,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
@@ -662,12 +661,6 @@ public ImmutableOpenMap<String, IndexTemplateMetaData> getTemplates() {
         return this.templates;
     }
 
-    public Map<String, ComponentTemplate> componentTemplates() {
-        return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
-            .map(ComponentTemplateMetadata::componentTemplates)
-            .orElse(Collections.emptyMap());
-    }
-
     public ImmutableOpenMap<String, Custom> customs() {
         return this.customs;
     }
@@ -1059,34 +1052,6 @@ public Builder templates(ImmutableOpenMap<String, IndexTemplateMetaData> templat
             return this;
         }
 
-        public Builder put(String name, ComponentTemplate componentTemplate) {
-            Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
-            // ಠ_ಠ at ImmutableOpenMap
-            Map<String, ComponentTemplate> existingTemplates =
-                Optional.ofNullable((ComponentTemplateMetadata) this.customs.get(ComponentTemplateMetadata.TYPE))
-                    .map(ctm -> new HashMap<>(ctm.componentTemplates()))
-                    .orElse(new HashMap<>());
-            existingTemplates.put(name, componentTemplate);
-            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(existingTemplates));
-            return this;
-        }
-
-        public Builder removeComponentTemplate(String name) {
-            // ಠ_ಠ at ImmutableOpenMap
-            Map<String, ComponentTemplate> existingTemplates =
-                Optional.ofNullable((ComponentTemplateMetadata) this.customs.get(ComponentTemplateMetadata.TYPE))
-                    .map(ctm -> new HashMap<>(ctm.componentTemplates()))
-                    .orElse(new HashMap<>());
-            existingTemplates.remove(name);
-            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(existingTemplates));
-            return this;
-        }
-
-        public Builder componentTemplates(Map<String, ComponentTemplate> componentTemplates) {
-            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(componentTemplates));
-            return this;
-        }
-
         public Custom getCustom(String type) {
             return customs.get(type);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index 6f8f680b10afd..fd4e7023cfee9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -318,19 +318,12 @@ public DiscoveryNode resolveNode(String node) {
     }
 
     /**
-     * Resolves a set of nodes according to the given sequence of node specifications. Implements the logic in various APIs that allow the
-     * user to run the action on a subset of the nodes in the cluster. See [Node specification] in the reference manual for full details.
-     *
-     * Works by tracking the current set of nodes and applying each node specification in sequence. The set starts out empty and each node
-     * specification may either add or remove nodes. For instance:
-     *
-     * - _local, _master and _all respectively add to the subset the local node, the currently-elected master, and all the nodes
-     * - node IDs, names, hostnames and IP addresses all add to the subset any nodes which match
-     * - a wildcard-based pattern of the form "attr*:value*" adds to the subset all nodes with a matching attribute with a matching value
-     * - role:true adds to the subset all nodes with a matching role
-     * - role:false removes from the subset all nodes with a matching role.
-     *
-     * An empty sequence of node specifications returns all nodes, since the corresponding actions run on all nodes by default.
+     * resolves a set of node "descriptions" to concrete and existing node ids. "descriptions" can be (resolved in this order):
+     * - "_local" or "_master" for the relevant nodes
+     * - a node id
+     * - a wild card pattern that will be matched against node names
+     * - a "attr:value" pattern, where attr can be a node role (master, data, ingest etc.) in which case the value can be true or false,
+     *   or a generic node attribute name in which case value will be treated as a wildcard and matched against the node attribute values.
      */
     public String[] resolveNodes(String... nodes) {
         if (nodes == null || nodes.length == 0) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
index 4f7da131977fb..67122cb3ff1b1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
@@ -137,6 +137,9 @@ public static AllocationCommands fromXContent(XContentParser parser) throws IOEx
             throw new ElasticsearchParseException("No commands");
         }
         if (token == XContentParser.Token.FIELD_NAME) {
+            if (!parser.currentName().equals("commands")) {
+                throw new ElasticsearchParseException("expected field name to be named [commands], got [{}] instead", parser.currentName());
+            }
             if (!parser.currentName().equals("commands")) {
                 throw new ElasticsearchParseException("expected field name to be named [commands], got [{}] instead", parser.currentName());
             }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
index a8a44ecb6e925..e78770ed2a85a 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
@@ -33,7 +33,6 @@
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
@@ -93,12 +92,11 @@ private Collection<SpanQuery> collectTerms(IndexReader reader, MultiTermQuery qu
                         continue;
                     }
 
-                    final TermsEnum termsEnum = getTermsEnum(query, terms, new AttributeSource());
+                    final TermsEnum termsEnum = getTermsEnum(query, terms, null);
                     assert termsEnum != null;
 
-                    if (termsEnum == TermsEnum.EMPTY) {
+                    if (termsEnum == TermsEnum.EMPTY)
                         continue;
-                    }
 
                     BytesRef bytes;
                     while ((bytes = termsEnum.next()) != null) {
diff --git a/server/src/main/java/org/elasticsearch/common/util/BitArray.java b/server/src/main/java/org/elasticsearch/common/util/BitArray.java
index 06844fd99ec8c..cfe2716e47e8e 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BitArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BitArray.java
@@ -32,10 +32,6 @@ public final class BitArray implements Releasable {
     private final BigArrays bigArrays;
     private LongArray bits;
 
-    /**
-     * Create the {@linkplain BitArray}.
-     * @param initialSize the initial size of underlying storage.
-     */
     public BitArray(int initialSize, BigArrays bigArrays) {
         this.bigArrays = bigArrays;
         this.bits = bigArrays.newLongArray(initialSize, true);
@@ -45,31 +41,21 @@ public BitArray(int initialSize, BigArrays bigArrays) {
      * Set the {@code index}th bit.
      */
     public void set(int index) {
-        int wordNum = wordNum(index);
-        bits = bigArrays.grow(bits, wordNum + 1);
-        bits.set(wordNum, bits.get(wordNum) | bitmask(index));
+        fill(index, true);
     }
 
     /**
      * Clear the {@code index}th bit.
      */
     public void clear(int index) {
-        int wordNum = wordNum(index);
-        if (wordNum >= bits.size()) {
-            /*
-             * No need to resize the array just to clear the bit because we'll
-             * initialize them to false when we grow the array anyway.
-             */
-            return;
-        }
-        bits.set(wordNum, bits.get(wordNum) & ~bitmask(index));
+        fill(index, false);
     }
 
     /**
      * Is the {@code index}th bit set?
      */
     public boolean get(int index) {
-        int wordNum = wordNum(index);
+        int wordNum = index >> 6;
         if (wordNum >= bits.size()) {
             /*
              * If the word is bigger than the array then it could *never* have
@@ -81,12 +67,12 @@ public boolean get(int index) {
         return (bits.get(wordNum) & bitmask) != 0;
     }
 
-    private static int wordNum(int index) {
-        return index >> 6;
-    }
-
-    private static long bitmask(int index) {
-        return 1L << index;
+    private void fill(int index, boolean bit) {
+        int wordNum = index >> 6;
+        bits = bigArrays.grow(bits,wordNum+1);
+        long bitmask = 1L << index;
+        long value = bit ? bits.get(wordNum) | bitmask : bits.get(wordNum) & ~bitmask;
+        bits.set(wordNum, value);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index 11f363f7ad706..a1319bcd59031 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index bd90f73054c94..de17eab3a4085 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -104,7 +104,6 @@ public abstract class Engine implements Closeable {
 
     public static final String SYNC_COMMIT_ID = "sync_id"; // TODO: Remove sync_id in 9.0
     public static final String HISTORY_UUID_KEY = "history_uuid";
-    public static final String FORCE_MERGE_UUID_KEY = "force_merge_uuid";
     public static final String MIN_RETAINED_SEQNO = "min_retained_seq_no";
     public static final String MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID = "max_unsafe_auto_id_timestamp";
 
@@ -1036,12 +1035,18 @@ public final void flush() throws EngineException {
      */
     public abstract void rollTranslogGeneration() throws EngineException;
 
+    /**
+     * Force merges to 1 segment
+     */
+    public void forceMerge(boolean flush) throws IOException {
+        forceMerge(flush, 1, false, false, false);
+    }
+
     /**
      * Triggers a forced merge on this engine
      */
     public abstract void forceMerge(boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                                    boolean upgrade, boolean upgradeOnlyAncientSegments,
-                                    @Nullable String forceMergeUUID) throws EngineException, IOException;
+                                        boolean upgrade, boolean upgradeOnlyAncientSegments) throws EngineException, IOException;
 
     /**
      * Snapshots the most recent index and returns a handle to it. If needed will try and "commit" the
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index a97f11af3213c..0a88123ba9413 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -184,12 +184,6 @@ public class InternalEngine extends Engine {
     @Nullable
     private final String historyUUID;
 
-    /**
-     * UUID value that is updated every time the engine is force merged.
-     */
-    @Nullable
-    private volatile String forceMergeUUID;
-
     public InternalEngine(EngineConfig engineConfig) {
         this(engineConfig, LocalCheckpointTracker::new);
     }
@@ -228,9 +222,7 @@ public InternalEngine(EngineConfig engineConfig) {
                 this.localCheckpointTracker = createLocalCheckpointTracker(localCheckpointTrackerSupplier);
                 writer = createWriter();
                 bootstrapAppendOnlyInfoFromWriter(writer);
-                final Map<String, String> commitData = commitDataAsMap(writer);
-                historyUUID = loadHistoryUUID(commitData);
-                forceMergeUUID = commitData.get(FORCE_MERGE_UUID_KEY);
+                historyUUID = loadHistoryUUID(writer);
                 indexWriter = writer;
             } catch (IOException | TranslogCorruptedException e) {
                 throw new EngineCreationFailureException(shardId, "failed to create engine", e);
@@ -566,12 +558,6 @@ public String getHistoryUUID() {
         return historyUUID;
     }
 
-    /** returns the force merge uuid for the engine */
-    @Nullable
-    public String getForceMergeUUID() {
-        return forceMergeUUID;
-    }
-
     /** Returns how many bytes we are currently moving from indexing buffer to segments on disk */
     @Override
     public long getWritingBytes() {
@@ -581,8 +567,8 @@ public long getWritingBytes() {
     /**
      * Reads the current stored history ID from the IW commit data.
      */
-    private String loadHistoryUUID(Map<String, String> commitData) {
-        final String uuid = commitData.get(HISTORY_UUID_KEY);
+    private String loadHistoryUUID(final IndexWriter writer) {
+        final String uuid = commitDataAsMap(writer).get(HISTORY_UUID_KEY);
         if (uuid == null) {
             throw new IllegalStateException("commit doesn't contain history uuid");
         }
@@ -1829,8 +1815,7 @@ final Map<BytesRef, VersionValue> getVersionMap() {
 
     @Override
     public void forceMerge(final boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                           final boolean upgrade, final boolean upgradeOnlyAncientSegments,
-                           final String forceMergeUUID) throws EngineException, IOException {
+                           final boolean upgrade, final boolean upgradeOnlyAncientSegments) throws EngineException, IOException {
         if (onlyExpungeDeletes && maxNumSegments >= 0) {
             throw new IllegalArgumentException("only_expunge_deletes and max_num_segments are mutually exclusive");
         }
@@ -1865,7 +1850,6 @@ public void forceMerge(final boolean flush, int maxNumSegments, boolean onlyExpu
                     indexWriter.maybeMerge();
                 } else {
                     indexWriter.forceMerge(maxNumSegments, true /* blocks and waits for merges*/);
-                    this.forceMergeUUID = forceMergeUUID;
                 }
                 if (flush) {
                     flush(false, true);
@@ -2313,16 +2297,12 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl
                  * {@link IndexWriter#commit()} call flushes all documents, we defer computation of the maximum sequence number to the time
                  * of invocation of the commit data iterator (which occurs after all documents have been flushed to Lucene).
                  */
-                final Map<String, String> commitData = new HashMap<>(7);
+                final Map<String, String> commitData = new HashMap<>(6);
                 commitData.put(Translog.TRANSLOG_UUID_KEY, translog.getTranslogUUID());
                 commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(localCheckpoint));
                 commitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(localCheckpointTracker.getMaxSeqNo()));
                 commitData.put(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp.get()));
                 commitData.put(HISTORY_UUID_KEY, historyUUID);
-                final String currentForceMergeUUID = forceMergeUUID;
-                if (currentForceMergeUUID != null) {
-                    commitData.put(FORCE_MERGE_UUID_KEY, currentForceMergeUUID);
-                }
                 commitData.put(Engine.MIN_RETAINED_SEQNO, Long.toString(softDeletesPolicy.getMinRetainedSeqNo()));
                 logger.trace("committing writer with commit data [{}]", commitData);
                 return commitData.entrySet().iterator();
diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
index 0e96689d6324b..7c39a3f11c4d5 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.seqno.SequenceNumbers;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
@@ -50,19 +49,18 @@
  */
 public final class NoOpEngine extends ReadOnlyEngine {
 
-    private final SegmentsStats segmentsStats;
-    private final DocsStats docsStats;
+    private final SegmentsStats stats;
 
     public NoOpEngine(EngineConfig config) {
         super(config, null, null, true, Function.identity());
-        this.segmentsStats = new SegmentsStats();
+        this.stats = new SegmentsStats();
         Directory directory = store.directory();
-        try (DirectoryReader reader = openDirectory(directory)) {
+        // Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
+        try (DirectoryReader reader = openDirectory(directory, false)) {
             for (LeafReaderContext ctx : reader.getContext().leaves()) {
                 SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
-                fillSegmentStats(segmentReader, true, segmentsStats);
+                fillSegmentStats(segmentReader, true, stats);
             }
-            this.docsStats = docsStats(reader);
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -119,7 +117,7 @@ public CacheHelper getReaderCacheHelper() {
     public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
         if (includeUnloadedSegments) {
             final SegmentsStats stats = new SegmentsStats();
-            stats.add(this.segmentsStats);
+            stats.add(this.stats);
             if (includeSegmentFileSizes == false) {
                 stats.clearFileSizes();
             }
@@ -129,11 +127,6 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
         }
     }
 
-    @Override
-    public DocsStats docStats() {
-        return docsStats;
-    }
-
     /**
      * This implementation will trim existing translog files using a {@link TranslogDeletionPolicy}
      * that retains nothing but the last translog generation from safe commit.
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index 62957e2c1c2c3..d84c7426447c8 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
 import org.apache.lucene.search.ReferenceManager;
@@ -35,6 +36,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
@@ -74,6 +76,7 @@ public class ReadOnlyEngine extends Engine {
     private final ElasticsearchReaderManager readerManager;
     private final IndexCommit indexCommit;
     private final Lock indexWriterLock;
+    private final DocsStats docsStats;
     private final RamAccountingRefreshListener refreshListener;
     private final SafeCommitInfo safeCommitInfo;
     private final CompletionStatsCache completionStatsCache;
@@ -116,6 +119,7 @@ public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats
                 this.indexCommit = Lucene.getIndexCommit(lastCommittedSegmentInfos, directory);
                 reader = wrapReader(open(indexCommit), readerWrapperFunction);
                 readerManager = new ElasticsearchReaderManager(reader, refreshListener);
+                this.docsStats = docsStats(lastCommittedSegmentInfos);
                 assert translogStats != null || obtainLock : "mutiple translogs instances should not be opened at the same time";
                 this.translogStats = translogStats != null ? translogStats : translogStats(config, lastCommittedSegmentInfos);
                 this.indexWriterLock = indexWriterLock;
@@ -178,6 +182,24 @@ protected DirectoryReader open(IndexCommit commit) throws IOException {
         return new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(commit, OFF_HEAP_READER_ATTRIBUTES), Lucene.SOFT_DELETES_FIELD);
     }
 
+    private DocsStats docsStats(final SegmentInfos lastCommittedSegmentInfos) {
+        long numDocs = 0;
+        long numDeletedDocs = 0;
+        long sizeInBytes = 0;
+        if (lastCommittedSegmentInfos != null) {
+            for (SegmentCommitInfo segmentCommitInfo : lastCommittedSegmentInfos) {
+                numDocs += segmentCommitInfo.info.maxDoc() - segmentCommitInfo.getDelCount() - segmentCommitInfo.getSoftDelCount();
+                numDeletedDocs += segmentCommitInfo.getDelCount() + segmentCommitInfo.getSoftDelCount();
+                try {
+                    sizeInBytes += segmentCommitInfo.sizeInBytes();
+                } catch (IOException e) {
+                    throw new UncheckedIOException("Failed to get size for [" + segmentCommitInfo.info.name + "]", e);
+                }
+            }
+        }
+        return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
+    }
+
     @Override
     protected void closeNoLock(String reason, CountDownLatch closedLatch) {
         if (isClosed.compareAndSet(false, true)) {
@@ -366,7 +388,7 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException {
 
     @Override
     public void forceMerge(boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                           boolean upgrade, boolean upgradeOnlyAncientSegments, String forceMergeUUID) {
+                           boolean upgrade, boolean upgradeOnlyAncientSegments) {
     }
 
     @Override
@@ -441,6 +463,11 @@ public void trimOperationsFromTranslog(long belowTerm, long aboveSeqNo) {
     public void maybePruneDeletes() {
     }
 
+    @Override
+    public DocsStats docStats() {
+        return docsStats;
+    }
+
     @Override
     public void updateMaxUnsafeAutoIdTimestamp(long newTimestamp) {
 
@@ -484,9 +511,13 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) {
             maxSeqNoOfUpdatesOnPrimary + ">" + getMaxSeqNoOfUpdatesOrDeletes();
     }
 
-    protected static DirectoryReader openDirectory(Directory directory) throws IOException {
+    protected static DirectoryReader openDirectory(Directory directory, boolean wrapSoftDeletes) throws IOException {
         final DirectoryReader reader = DirectoryReader.open(directory, OFF_HEAP_READER_ATTRIBUTES);
-        return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
+        if (wrapSoftDeletes) {
+            return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
+        } else {
+            return reader;
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
index 06352640dc162..27088382f2025 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
@@ -21,6 +21,8 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.search.SortedSetSelector;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
@@ -52,7 +54,20 @@ public BinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exce
     public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested,
             boolean reverse) {
         XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
-        return new SortField(getFieldName(), source, reverse);
+        /**
+         * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
+         * returns a custom sort field otherwise.
+         */
+        if (nested != null ||
+                (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) ||
+                (source.sortMissingFirst(missingValue) == false && source.sortMissingLast(missingValue) == false)) {
+            return new SortField(getFieldName(), source, reverse);
+        }
+        SortField sortField = new SortedSetSortField(fieldName, reverse,
+            sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN);
+        sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ?
+            SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST);
+        return sortField;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
index 4e6b04a72418a..87d489d04b556 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -97,11 +97,6 @@ public Instant toInstant(long value) {
             public Instant clampToValidRange(Instant instant) {
                 return instant;
             }
-
-            @Override
-            public long parsePointAsMillis(byte[] value) {
-                return LongPoint.decodeDimension(value, 0);
-            }
         },
         NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS) {
             @Override
@@ -118,11 +113,6 @@ public Instant toInstant(long value) {
             public Instant clampToValidRange(Instant instant) {
                 return DateUtils.clampToNanosRange(instant);
             }
-
-            @Override
-            public long parsePointAsMillis(byte[] value) {
-                return DateUtils.toMilliSeconds(LongPoint.decodeDimension(value, 0));
-            }
         };
 
         private final String type;
@@ -151,17 +141,8 @@ NumericType numericType() {
          */
         public abstract Instant toInstant(long value);
 
-        /**
-         * Return the instant that this range can represent that is closest to
-         * the provided instant.
-         */
         public abstract Instant clampToValidRange(Instant instant);
 
-        /**
-         * Decode the points representation of this field as milliseconds.
-         */
-        public abstract long parsePointAsMillis(byte[] value);
-
         public static Resolution ofOrdinal(int ord) {
             for (Resolution resolution : values()) {
                 if (ord == resolution.ordinal()) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index b83427a53a3ed..7fde4359742e3 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -204,7 +204,6 @@ protected KeywordFieldType(KeywordFieldType ref) {
             this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace;
         }
 
-        @Override
         public KeywordFieldType clone() {
             return new KeywordFieldType(this);
         }
@@ -322,7 +321,8 @@ protected KeywordFieldMapper(String simpleName, MappedFieldType fieldType, Mappe
 
     /** Values that have more chars than the return value of this method will
      *  be skipped at parsing time. */
-    public int ignoreAbove() {
+    // pkg-private for testing
+    int ignoreAbove() {
         return ignoreAbove;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index d675735834f7b..b2c536a900e33 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -120,6 +120,24 @@ public IndexableField[] getFields(String name) {
             return f.toArray(new IndexableField[f.size()]);
         }
 
+        /**
+         * Returns an array of values of the field specified as the method parameter.
+         * This method returns an empty array when there are no
+         * matching fields.  It never returns null.
+         * If you want the actual numeric field instances back, use {@link #getFields}.
+         * @param name the name of the field
+         * @return a <code>String[]</code> of field values
+         */
+        public final String[] getValues(String name) {
+            List<String> result = new ArrayList<>();
+            for (IndexableField field : fields) {
+                if (field.name().equals(name) && field.stringValue() != null) {
+                    result.add(field.stringValue());
+                }
+            }
+            return result.toArray(new String[result.size()]);
+        }
+
         public IndexableField getField(String name) {
             for (IndexableField field : fields) {
                 if (field.name().equals(name)) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index 05bf6b61d1de1..4ddda3df0af2d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -21,6 +21,8 @@
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
@@ -29,7 +31,6 @@
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -37,8 +38,6 @@
 import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 
@@ -48,8 +47,6 @@
  * can be implemented. */
 public abstract class StringFieldType extends TermBasedFieldType {
 
-    private static final Pattern WILDCARD_PATTERN = Pattern.compile("(\\\\.)|([?*]+)");
-
     public StringFieldType() {}
 
     protected StringFieldType(MappedFieldType ref) {
@@ -95,41 +92,16 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
 
     @Override
     public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        failIfNotIndexed();
+        Query termQuery = termQuery(value, context);
+        if (termQuery instanceof MatchNoDocsQuery || termQuery instanceof MatchAllDocsQuery) {
+            return termQuery;
+        }
+
         if (context.allowExpensiveQueries() == false) {
             throw new ElasticsearchException("[wildcard] queries cannot be executed when '" +
                     ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false.");
         }
-
-        Term term;
-        if (searchAnalyzer() != null) {
-            // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
-            // is a char_filter that would otherwise remove them
-            Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
-            BytesRefBuilder sb = new BytesRefBuilder();
-            int last = 0;
-
-            while (wildcardMatcher.find()) {
-                if (wildcardMatcher.start() > 0) {
-                    String chunk = value.substring(last, wildcardMatcher.start());
-
-                    BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
-                    sb.append(normalized);
-                }
-                // append the matched group - without normalizing
-                sb.append(new BytesRef(wildcardMatcher.group()));
-
-                last = wildcardMatcher.end();
-            }
-            if (last < value.length()) {
-                String chunk = value.substring(last);
-                BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
-                sb.append(normalized);
-            }
-            term = new Term(name(), sb.toBytesRef());
-        } else {
-            term = new Term(name(), indexedValueForSearch(value));
-        }
+        Term term = MappedFieldType.extractTerm(termQuery);
 
         WildcardQuery query = new WildcardQuery(term);
         QueryParsers.setRewriteMethod(query, method);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index 1f9ccfef04330..895243d43faf3 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
 import org.apache.lucene.search.AutomatonQuery;
@@ -424,13 +425,13 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
 
         public IntervalsSource intervals(BytesRef term) {
             if (term.length > maxChars) {
-                return Intervals.prefix(term);
+                return XIntervals.prefix(term);
             }
             if (term.length >= minChars) {
                 return Intervals.fixField(name(), Intervals.term(term));
             }
             String wildcardTerm = term.utf8ToString() + "?".repeat(Math.max(0, minChars - term.length));
-            return Intervals.or(Intervals.fixField(name(), Intervals.wildcard(new BytesRef(wildcardTerm))), Intervals.term(term));
+            return Intervals.or(Intervals.fixField(name(), XIntervals.wildcard(new BytesRef(wildcardTerm))), Intervals.term(term));
         }
 
         @Override
@@ -674,7 +675,7 @@ public IntervalsSource intervals(String text, int maxGaps, boolean ordered,
                 if (prefixFieldType != null) {
                     return prefixFieldType.intervals(normalizedTerm);
                 }
-                return Intervals.prefix(normalizedTerm);
+                return XIntervals.prefix(normalizedTerm);
             }
             IntervalBuilder builder = new IntervalBuilder(name(), analyzer == null ? searchAnalyzer() : analyzer);
             return builder.analyzeText(text, maxGaps, ordered);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
index d32d4fa9d3f11..e5c33b83f24e5 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
@@ -30,11 +30,13 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -88,7 +90,7 @@ public MetadataFieldMapper getDefault(ParserContext context) {
         }
     }
 
-    public static final class TypeFieldType extends ConstantFieldType {
+    public static final class TypeFieldType extends StringFieldType {
 
         TypeFieldType() {
         }
@@ -119,8 +121,61 @@ public ValuesSourceType getValuesSourceType() {
         }
 
         @Override
-        protected boolean matches(String pattern, QueryShardContext context) {
-            return pattern.equals(MapperService.SINGLE_MAPPING_NAME);
+        public boolean isSearchable() {
+            return true;
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            return new MatchAllDocsQuery();
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            return termsQuery(Arrays.asList(value), context);
+        }
+
+        @Override
+        public Query termsQuery(List<?> values, QueryShardContext context) {
+            DocumentMapper mapper = context.getMapperService().documentMapper();
+            if (mapper == null) {
+                return new MatchNoDocsQuery("No types");
+            }
+            BytesRef indexType = indexedValueForSearch(mapper.type());
+            if (values.stream()
+                    .map(this::indexedValueForSearch)
+                    .anyMatch(indexType::equals)) {
+                if (context.getMapperService().hasNested()) {
+                    // type filters are expected not to match nested docs
+                    return Queries.newNonNestedFilter();
+                } else {
+                    return new MatchAllDocsQuery();
+                }
+            } else {
+                return new MatchNoDocsQuery("Type list does not contain the index type");
+            }
+        }
+
+        @Override
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
+            Query result = new MatchAllDocsQuery();
+            String type = context.getMapperService().documentMapper().type();
+            if (type != null) {
+                BytesRef typeBytes = new BytesRef(type);
+                if (lowerTerm != null) {
+                    int comp = indexedValueForSearch(lowerTerm).compareTo(typeBytes);
+                    if (comp > 0 || (comp == 0 && includeLower == false)) {
+                        result = new MatchNoDocsQuery("[_type] was lexicographically smaller than lower bound of range");
+                    }
+                }
+                if (upperTerm != null) {
+                    int comp = indexedValueForSearch(upperTerm).compareTo(typeBytes);
+                    if (comp < 0 || (comp == 0 && includeUpper == false)) {
+                        result = new MatchNoDocsQuery("[_type] was lexicographically greater than upper bound of range");
+                    }
+                }
+            }
+            return result;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
index 13c60c0a49f5e..eefbc0cb3ae55 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.FilteredIntervalsSource;
 import org.apache.lucene.queries.intervals.IntervalIterator;
 import org.apache.lucene.queries.intervals.Intervals;
@@ -654,12 +655,13 @@ public IntervalsSource getSource(QueryShardContext context, MappedFieldType fiel
                     analyzer = fieldType.searchAnalyzer();
                 }
                 BytesRef normalizedTerm = analyzer.normalize(useField, pattern);
-                source = Intervals.fixField(useField, Intervals.wildcard(normalizedTerm));
+                // TODO Intervals.wildcard() should take BytesRef
+                source = Intervals.fixField(useField, XIntervals.wildcard(normalizedTerm));
             }
             else {
                 checkPositions(fieldType);
                 BytesRef normalizedTerm = analyzer.normalize(fieldType.name(), pattern);
-                source = Intervals.wildcard(normalizedTerm);
+                source = XIntervals.wildcard(normalizedTerm);
             }
             return source;
         }
@@ -796,7 +798,7 @@ public IntervalsSource getSource(QueryShardContext context, MappedFieldType fiel
             FuzzyQuery fq = new FuzzyQuery(new Term(fieldType.name(), normalizedTerm),
                 fuzziness.asDistance(term), prefixLength, 128, transpositions);
             CompiledAutomaton[] automata = fq.getAutomata();
-            source = Intervals.multiterm(automata[automata.length - 1], term);
+            source = XIntervals.multiterm(automata[automata.length - 1], term);
             if (useField != null) {
                 source = Intervals.fixField(useField, source);
             }
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 56707bc198beb..4c7b8f568aff3 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1073,7 +1073,7 @@ public void forceMerge(ForceMergeRequest forceMerge) throws IOException {
         }
         Engine engine = getEngine();
         engine.forceMerge(forceMerge.flush(), forceMerge.maxNumSegments(),
-            forceMerge.onlyExpungeDeletes(), false, false, forceMerge.forceMergeUUID());
+            forceMerge.onlyExpungeDeletes(), false, false);
     }
 
     /**
@@ -1089,7 +1089,7 @@ public org.apache.lucene.util.Version upgrade(UpgradeRequest upgrade) throws IOE
         final Engine engine = getEngine();
         engine.forceMerge(true,  // we need to flush at the end to make sure the upgrade is durable
             Integer.MAX_VALUE, // we just want to upgrade the segments, not actually optimize to a single segment
-            false, true, upgrade.upgradeOnlyAncientSegments(), null);
+            false, true, upgrade.upgradeOnlyAncientSegments());
         org.apache.lucene.util.Version version = minimumCompatibleVersion();
         if (logger.isTraceEnabled()) {
             logger.trace("upgraded segments for {} from version {} to version {}", shardId, previousVersion, version);
diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index e6301371520a5..b84b6aa4c1e49 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -136,7 +135,7 @@ void recoverFromLocalShards(Consumer<MappingMetaData> mappingUpdateConsumer, fin
                     // just trigger a merge to do housekeeping on the
                     // copied segments - we will also see them in stats etc.
                     indexShard.getEngine().forceMerge(false, -1, false,
-                        false, false, UUIDs.randomBase64UUID());
+                        false, false);
                     return true;
                 } catch (IOException ex) {
                     throw new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", ex);
diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 65df5e2b35268..400e5355c0942 100644
--- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
@@ -57,7 +56,6 @@
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -315,7 +313,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
             else {
                 seenFields.add(field.name());
             }
-            String[] values = getValues(doc.getFields(field.name()));
+            String[] values = doc.getValues(field.name());
             documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values)));
         }
         return generateTermVectors(indexShard,
@@ -323,25 +321,6 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
                 request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
-    /**
-     * Returns an array of values of the field specified as the method parameter.
-     * This method returns an empty array when there are no
-     * matching fields.  It never returns null.
-     * @param fields The <code>IndexableField</code> to get the values from
-     * @return a <code>String[]</code> of field values
-     */
-    public static String[] getValues(IndexableField[] fields) {
-        List<String> result = new ArrayList<>();
-        for (IndexableField field : fields) {
-            if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) {
-                result.add(field.binaryValue().utf8ToString());
-            } else if (field.fieldType() instanceof StringFieldType) {
-                result.add(field.stringValue());
-            }
-        }
-        return result.toArray(new String[0]);
-    }
-
     private static ParsedDocument parseDocument(IndexShard indexShard, String index, BytesReference doc,
                                                 XContentType xContentType, String routing) {
         MapperService mapperService = indexShard.mapperService();
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index 90f20811c3dc9..e03b0c98e0e98 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -21,7 +21,6 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
@@ -53,7 +52,6 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, final Node
         final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?
                 new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias"))) :
                 new GetAliasesRequest();
-        getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
         getAliasesRequest.local(request.paramAsBoolean("local", getAliasesRequest.local()));
 
         return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestResponseListener<GetAliasesResponse>(channel) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index d760d56e8d542..a70fcec89b0c8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -90,7 +90,7 @@ protected void documentation(StringBuilder sb) {
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.strictExpand());
+        final IndicesOptions indicesOptions = IndicesOptions.strictExpand();
         final boolean local = request.paramAsBoolean("local", false);
         final TimeValue masterNodeTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT);
         final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false);
@@ -109,12 +109,6 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) {
                     final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(request, 4, listener);
                     groupedListener.onResponse(getSettingsResponse);
 
-                    // The list of indices that will be returned is determined by the indices returned from the Get Settings call.
-                    // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the
-                    // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so
-                    // force the IndicesOptions for all the sub-requests to be as inclusive as possible.
-                    final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden();
-
                     // Indices that were successfully resolved during the get settings request might be deleted when the subsequent cluster
                     // state, cluster health and indices stats requests execute. We have to distinguish two cases:
                     // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the
@@ -123,11 +117,11 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) {
                     //    fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
                     // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve the
                     // index names with the same indices options that we used for the initial cluster state request (strictExpand).
-                    sendIndicesStatsRequest(indices, subRequestIndicesOptions, includeUnloadedSegments, client,
+                    sendIndicesStatsRequest(indices, indicesOptions, includeUnloadedSegments, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
-                    sendClusterStateRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
+                    sendClusterStateRequest(indices, indicesOptions, local, masterNodeTimeout, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
-                    sendClusterHealthRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
+                    sendClusterHealthRequest(indices, indicesOptions, local, masterNodeTimeout, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
                 }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index b95aaf4b86018..266ff0321033a 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -164,7 +164,7 @@ public static void parseSearchRequest(SearchRequest searchRequest, RestRequest r
         searchRequest.routing(request.param("routing"));
         searchRequest.preference(request.param("preference"));
         searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
-        searchRequest.setCcsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", searchRequest.isCcsMinimizeRoundtrips()));
+        searchRequest.setCcsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", true));
 
         checkRestTotalHits(request, searchRequest);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index bed38deb28a29..24ab7f5e8bf43 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -62,7 +62,6 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
@@ -83,7 +82,7 @@
 
 final class DefaultSearchContext extends SearchContext {
 
-    private final SearchContextId id;
+    private final long id;
     private final ShardSearchRequest request;
     private final SearchShardTarget shardTarget;
     private final LongSupplier relativeTimeSupplier;
@@ -153,7 +152,7 @@ final class DefaultSearchContext extends SearchContext {
     private final QueryShardContext queryShardContext;
     private final FetchPhase fetchPhase;
 
-    DefaultSearchContext(SearchContextId id, ShardSearchRequest request, SearchShardTarget shardTarget,
+    DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
                          Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService,
                          IndexShard indexShard, BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout,
                          FetchPhase fetchPhase) throws IOException {
@@ -296,7 +295,7 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
     }
 
     @Override
-    public SearchContextId id() {
+    public long id() {
         return this.id;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
index 7f681bfa47c92..b09c8983c1394 100644
--- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -21,7 +21,6 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -202,13 +201,6 @@ public DateTime(StreamInput in) throws IOException {
             String zoneId = in.readString();
             this.timeZone = ZoneId.of(zoneId);
             this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt());
-            if (in.getVersion().onOrAfter(Version.V_7_7_0) && in.getVersion().before(Version.V_8_0_0)) {
-                /* when deserialising from 7.7+ nodes expect a flag indicating if a pattern is of joda style
-                   This is only used to support joda style indices in 7.x, in 8 we no longer support this.
-                   All indices in 8 should use java style pattern. Hence we can ignore this flag.
-                */
-                in.readBoolean();
-            }
         }
 
         @Override
@@ -221,13 +213,6 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(formatter.pattern());
             out.writeString(timeZone.getId());
             out.writeVInt(resolution.ordinal());
-            if (out.getVersion().onOrAfter(Version.V_7_7_0) && out.getVersion().before(Version.V_8_0_0)) {
-                /* when serializing to 7.7+  send out a flag indicating if a pattern is of joda style
-                   This is only used to support joda style indices in 7.x, in 8 we no longer support this.
-                   All indices in 8 should use java style pattern. Hence this flag is always false.
-                */
-                out.writeBoolean(false);
-            }
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java b/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
index 03ac85a8d81ac..9e8d41e2ae29c 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
@@ -23,21 +23,20 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.internal.SearchContextId;
 
 import java.io.IOException;
 
 public class SearchContextMissingException extends ElasticsearchException {
 
-    private final SearchContextId contextId;
+    private final long id;
 
-    public SearchContextMissingException(SearchContextId contextId) {
-        super("No search context found for id [" + contextId.getId() + "]");
-        this.contextId = contextId;
+    public SearchContextMissingException(long id) {
+        super("No search context found for id [" + id + "]");
+        this.id = id;
     }
 
-    public SearchContextId contextId() {
-        return this.contextId;
+    public long id() {
+        return this.id;
     }
 
     @Override
@@ -47,12 +46,12 @@ public RestStatus status() {
 
     public SearchContextMissingException(StreamInput in) throws IOException{
         super(in);
-        contextId = new SearchContextId(in);
+        id = in.readLong();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        contextId.writeTo(out);
+        out.writeLong(id);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
index 15fab25f01b6b..77af879831e02 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.TransportResponse;
 
@@ -40,7 +39,7 @@ public abstract class SearchPhaseResult extends TransportResponse {
 
     private SearchShardTarget searchShardTarget;
     private int shardIndex = -1;
-    protected SearchContextId contextId;
+    protected long requestId;
 
     protected SearchPhaseResult() {
 
@@ -51,10 +50,10 @@ protected SearchPhaseResult(StreamInput in) throws IOException {
     }
 
     /**
-     * Returns the search context ID that is used to reference the search context on the executing node
+     * Returns the results request ID that is used to reference the search context on the executing node
      */
-    public SearchContextId getContextId() {
-        return contextId;
+    public long getRequestId() {
+        return requestId;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index c7fa46bf9a353..49b560dcd5b58 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -84,7 +83,6 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
@@ -286,7 +284,7 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
     }
 
     protected void putContext(SearchContext context) {
-        final SearchContext previous = activeContexts.put(context.id().getId(), context);
+        final SearchContext previous = activeContexts.put(context.id(), context);
         assert previous == null;
     }
 
@@ -414,8 +412,8 @@ private void onMatchNoDocs(SearchRewriteContext rewriteContext, ActionListener<S
         listener.onResponse(QuerySearchResult.nullInstance());
     }
 
-    private <T> void runAsync(SearchContextId contextId, Supplier<T> executable, ActionListener<T> listener) {
-        getExecutor(contextId).execute(ActionRunnable.supply(listener, executable::get));
+    private <T> void runAsync(long id, Supplier<T> executable, ActionListener<T> listener) {
+        getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
     }
 
     private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext, SearchShardTask task) throws Exception {
@@ -470,8 +468,8 @@ private QueryFetchSearchResult executeFetchPhase(SearchContext context, long aft
     public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final SearchContext context = findContext(request.contextId(), request);
+        runAsync(request.id(), () -> {
+            final SearchContext context = findContext(request.id(), request);
             context.incRef();
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 context.setTask(task);
@@ -492,8 +490,8 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final SearchContext context = findContext(request.contextId(), request);
+        runAsync(request.id(), () -> {
+            final SearchContext context = findContext(request.id(), request);
             context.setTask(task);
             context.incRef();
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
@@ -528,11 +526,10 @@ private boolean fetchPhaseShouldFreeContext(SearchContext context) {
         }
     }
 
-
-    final Executor getExecutor(SearchContextId contextId) {
-        SearchContext context = getContext(contextId);
+    final Executor getExecutor(long id) {
+        SearchContext context = activeContexts.get(id);
         if (context == null) {
-            throw new SearchContextMissingException(contextId);
+            throw new SearchContextMissingException(id);
         }
         return getExecutor(context.indexShard());
     }
@@ -544,8 +541,8 @@ private Executor getExecutor(IndexShard indexShard) {
 
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final SearchContext context = findContext(request.contextId(), request);
+        runAsync(request.id(), () -> {
+            final SearchContext context = findContext(request.id(), request);
             context.setTask(task);
             context.incRef();
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)){
@@ -566,8 +563,8 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final SearchContext context = findContext(request.contextId(), request);
+        runAsync(request.id(), () -> {
+            final SearchContext context = findContext(request.id(), request);
             context.incRef();
             try {
                 context.setTask(task);
@@ -579,7 +576,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
                     fetchPhase.execute(context);
                     if (fetchPhaseShouldFreeContext(context)) {
-                        freeContext(request.contextId());
+                        freeContext(request.id());
                     } else {
                         contextProcessedSuccessfully(context);
                     }
@@ -596,21 +593,10 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
         }, listener);
     }
 
-    private SearchContext getContext(SearchContextId contextId) {
-        final SearchContext context = activeContexts.get(contextId.getId());
-        if (context == null) {
-            return null;
-        }
-        if (context.id().getReaderId().equals(contextId.getReaderId()) || contextId.getReaderId().isEmpty()) {
-            return context;
-        }
-        return null;
-    }
-
-    private SearchContext findContext(SearchContextId contextId, TransportRequest request) throws SearchContextMissingException {
-        final SearchContext context = getContext(contextId);
+    private SearchContext findContext(long id, TransportRequest request) throws SearchContextMissingException {
+        SearchContext context = activeContexts.get(id);
         if (context == null) {
-            throw new SearchContextMissingException(contextId);
+            throw new SearchContextMissingException(id);
         }
 
         SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
@@ -642,6 +628,7 @@ private void onNewContext(SearchContext context) {
         boolean success = false;
         try {
             if (context.scrollContext() != null) {
+                openScrollContexts.incrementAndGet();
                 context.indexShard().getSearchOperationListener().onNewScrollContext(context);
             }
             context.indexShard().getSearchOperationListener().onNewContext(context);
@@ -660,15 +647,14 @@ private void onNewContext(SearchContext context) {
     final SearchContext createContext(SearchRewriteContext rewriteContext) throws IOException {
         final DefaultSearchContext context = createSearchContext(rewriteContext, defaultSearchTimeout);
         try {
+            if (rewriteContext.request != null && openScrollContexts.get() >= maxOpenScrollContext) {
+                throw new ElasticsearchException(
+                    "Trying to create too many scroll contexts. Must be less than or equal to: [" +
+                        maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
+                        + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
+            }
             final ShardSearchRequest request = rewriteContext.request;
             if (request.scroll() != null) {
-                context.addReleasable(openScrollContexts::decrementAndGet, Lifetime.CONTEXT);
-                if (openScrollContexts.incrementAndGet() > maxOpenScrollContext) {
-                    throw new ElasticsearchException(
-                        "Trying to create too many scroll contexts. Must be less than or equal to: [" +
-                            maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
-                            + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
-                }
                 context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
             }
@@ -718,17 +704,8 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon
             IndexShard indexShard = indexService.getShard(request.shardId().getId());
             SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
                 indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
-            // TODO: If no changes are made since the last commit, and the searcher is opened from that commit, then we can use the
-            //  commit_id as the context_id. And if the local checkpoint and max_seq_no of that commit equal the global checkpoint,
-            //  then we can use a combination of history_uuid and one of these values as a **weaker** context_id.
-            //  Reader contexts with the same commit_id can be replaced at any time, as the Lucene doc ids are the same.
-            //  Reader contexts with the same seq_id, however, can't be replaced between the query and fetch phase because
-            //  the Lucene doc ids can be different.
-            final String readerId = UUIDs.base64UUID();
-            DefaultSearchContext searchContext = new DefaultSearchContext(
-                new SearchContextId(readerId, idGenerator.incrementAndGet()),
-                request, shardTarget, searcher, clusterService, indexService, indexShard, bigArrays,
-                threadPool::relativeTimeInMillis, timeout, fetchPhase);
+            DefaultSearchContext searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget,
+                searcher, clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
             success = true;
             return searchContext;
         } finally {
@@ -741,7 +718,6 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon
         }
     }
 
-
     private void freeAllContextForIndex(Index index) {
         assert index != null;
         for (SearchContext ctx : activeContexts.values()) {
@@ -751,23 +727,23 @@ private void freeAllContextForIndex(Index index) {
         }
     }
 
-    public boolean freeContext(SearchContextId contextId) {
-        if (getContext(contextId) != null) {
-            try (SearchContext context = removeContext(contextId.getId())) {
-                if (context != null) {
-                    onFreeContext(context);
-                    return true;
-                }
+
+    public boolean freeContext(long id) {
+        try (SearchContext context = removeContext(id)) {
+            if (context != null) {
+                onFreeContext(context);
+                return true;
             }
+            return false;
         }
-        return false;
     }
 
     private void onFreeContext(SearchContext context) {
         assert context.refCount() > 0 : " refCount must be > 0: " + context.refCount();
-        assert activeContexts.containsKey(context.id().getId()) == false;
+        assert activeContexts.containsKey(context.id()) == false;
         context.indexShard().getSearchOperationListener().onFreeContext(context);
         if (context.scrollContext() != null) {
+            openScrollContexts.decrementAndGet();
             context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index 77833c98cae0e..021d6872e3325 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -34,7 +34,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Function;
 import java.util.function.IntConsumer;
 
 /**
@@ -128,29 +127,6 @@ public String getName() {
         return name;
     }
 
-    /**
-     * Rewrite the sub-aggregations in the buckets in this aggregation.
-     * Returns a copy of this {@linkplain InternalAggregation} with the
-     * rewritten buckets, or, if there aren't any modifications to
-     * the buckets then this method will return this aggregation. Either
-     * way, it doesn't modify this aggregation.
-     * <p>
-     * Implementers of this should call the {@code rewriter} once per bucket
-     * with its {@linkplain InternalAggregations}. The {@code rewriter}
-     * should return {@code null} if it doen't have any rewriting to do or
-     * it should return a new {@linkplain InternalAggregations} to make
-     * changs.
-     * <p>
-     * The default implementation throws an exception because most
-     * aggregations don't <strong>have</strong> buckets in them. It
-     * should be overridden by aggregations that contain buckets. Implementers
-     * should respect the description above.
-     */
-    public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations, InternalAggregations> rewriter) {
-        throw new IllegalStateException(
-                "Aggregation [" + getName() + "] must be a bucket aggregation but was [" + getWriteableName() + "]");
-    }
-
     /**
      * Creates the output from all pipeline aggs that this aggregation is associated with.  Should only
      * be called after all aggregations have been fully reduced
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index 07a992d197cd1..3f18d34fd01b4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -85,15 +85,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeNamedWriteableList(topLevelPipelineAggregators);
     }
 
-    /**
-     * Make a mutable copy of the aggregation results.
-     * <p>
-     * IMPORTANT: The copy doesn't include any pipeline aggregations, if there are any.
-     */
-    public List<InternalAggregation> copyResults() {
-        return new ArrayList<>(getInternalAggregations());
-    }
-
     /**
      * Returns the top-level pipeline aggregators.
      * Note that top-level pipeline aggregators become normal aggregation once the final reduction has been performed, after which they
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index 61176fbf29f82..9600c6ae86611 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -29,7 +29,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 
 public abstract class InternalMultiBucketAggregation<A extends InternalMultiBucketAggregation,
             B extends InternalMultiBucketAggregation.InternalBucket>
@@ -155,23 +154,6 @@ public final InternalAggregation reducePipelines(InternalAggregation reducedAggs
         return super.reducePipelines(create(materializedBuckets), reduceContext);
     }
 
-    @Override
-    public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations, InternalAggregations> rewriter) {
-        boolean modified = false;
-        List<B> newBuckets = new ArrayList<>();
-        for (B bucket : getBuckets()) {
-            InternalAggregations rewritten = rewriter.apply((InternalAggregations) bucket.getAggregations());
-            if (rewritten == null) {
-                newBuckets.add(bucket);
-                continue;
-            }
-            modified = true;
-            B newBucket = createBucket(rewritten, bucket);
-            newBuckets.add(newBucket);
-        }
-        return modified ? create(newBuckets) : this;
-    }
-
     private List<B> reducePipelineBuckets(ReduceContext reduceContext) {
         List<B> reducedBuckets = new ArrayList<>();
         for (B bucket : getBuckets()) {
@@ -210,5 +192,6 @@ public Object getProperty(String containingAggName, List<String> path) {
             }
             return aggregation.getProperty(path.subList(1, path.size()));
         }
+
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
index 437cd1466d688..68a66318b4511 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
@@ -33,7 +33,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Function;
 
 /**
  * A base class for all the single bucket aggregations.
@@ -170,15 +169,6 @@ public final double sortValue(AggregationPath.PathElement head, Iterator<Aggrega
         return aggregations.sortValue(head, tail);
     }
 
-    @Override
-    public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations, InternalAggregations> rewriter) {
-        InternalAggregations rewritten = rewriter.apply(aggregations);
-        if (rewritten == null) {
-            return this;
-        }
-        return create(rewritten);
-    }
-
     @Override
     public boolean equals(Object obj) {
         if (this == obj) return true;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
index 801438bc84f7f..78181e3a3366a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
@@ -83,7 +84,7 @@ public long getDocCount() {
         }
 
         @Override
-        public InternalAggregations getAggregations() {
+        public Aggregations getAggregations() {
             return aggregations;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
index b4c72b8529c0f..0e01615492939 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -202,8 +202,7 @@ private Sort buildIndexSortPrefix(LeafReaderContext context) throws IOException
             return null;
         }
         List<SortField> sortFields = new ArrayList<>();
-        int end = Math.min(indexSort.getSort().length, sourceConfigs.length);
-        for (int i = 0; i < end; i++) {
+        for (int i = 0; i < indexSort.getSort().length; i++) {
             CompositeValuesSourceConfig sourceConfig = sourceConfigs[i];
             SingleDimensionValuesSource<?> source = sources[i];
             SortField indexSortField = indexSort.getSort()[i];
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
index 93dcdf7be3bda..d71ed3c3bd97d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
@@ -269,8 +269,7 @@ SortedDocsProducer createSortedDocsProducerOrNull(IndexReader reader, Query quer
             }
             return new PointsSortedDocsProducer(fieldType.name(), toBucketFunction, lowerPoint, upperPoint);
         } else if (fieldType instanceof DateFieldMapper.DateFieldType) {
-            ToLongFunction<byte[]> decode = ((DateFieldMapper.DateFieldType) fieldType).resolution()::parsePointAsMillis;
-            ToLongFunction<byte[]> toBucketFunction = value -> rounding.applyAsLong(decode.applyAsLong(value));
+            final ToLongFunction<byte[]> toBucketFunction = (value) -> rounding.applyAsLong(LongPoint.decodeDimension(value, 0));
             return new PointsSortedDocsProducer(fieldType.name(), toBucketFunction, lowerPoint, upperPoint);
         } else {
             return null;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index ba786f9414778..d8f74b1b8e42f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -255,49 +255,31 @@ public InternalAggregation reduce(List<InternalAggregation> aggregations, Reduce
             }
         }
 
-        final B[] list;
-        if (reduceContext.isFinalReduce()) {
-            final int size = Math.min(requiredSize, buckets.size());
-            final BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator());
-            for (List<B> sameTermBuckets : buckets.values()) {
-                final B b = reduceBucket(sameTermBuckets, reduceContext);
-                if (sumDocCountError == -1) {
-                    b.docCountError = -1;
-                } else {
-                    b.docCountError += sumDocCountError;
-                }
-                if (b.docCount >= minDocCount) {
-                    B removed = ordered.insertWithOverflow(b);
-                    if (removed != null) {
-                        otherDocCount += removed.getDocCount();
-                        reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
-                    } else {
-                        reduceContext.consumeBucketsAndMaybeBreak(1);
-                    }
-                } else {
-                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(b));
-                }
-            }
-            list = createBucketsArray(ordered.size());
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = ordered.pop();
+        final int size = reduceContext.isFinalReduce() == false ? buckets.size() : Math.min(requiredSize, buckets.size());
+        final BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator());
+        for (List<B> sameTermBuckets : buckets.values()) {
+            final B b = reduceBucket(sameTermBuckets, reduceContext);
+            if (sumDocCountError == -1) {
+                b.docCountError = -1;
+            } else {
+                b.docCountError += sumDocCountError;
             }
-        } else {
-            // keep all buckets on partial reduce
-            // TODO: we could prune the buckets when sorting by key
-            list = createBucketsArray(buckets.size());
-            int pos = 0;
-            for (List<B> sameTermBuckets : buckets.values()) {
-                final B b = reduceBucket(sameTermBuckets, reduceContext);
-                reduceContext.consumeBucketsAndMaybeBreak(1);
-                if (sumDocCountError == -1) {
-                    b.docCountError = -1;
+            if (b.docCount >= minDocCount || reduceContext.isFinalReduce() == false) {
+                B removed = ordered.insertWithOverflow(b);
+                if (removed != null) {
+                    otherDocCount += removed.getDocCount();
+                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
                 } else {
-                    b.docCountError += sumDocCountError;
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
                 }
-                list[pos++] = b;
+            } else {
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(b));
             }
         }
+        B[] list = createBucketsArray(ordered.size());
+        for (int i = ordered.size() - 1; i >= 0; i--) {
+            list[i] = ordered.pop();
+        }
         long docCountError;
         if (sumDocCountError == -1) {
             docCountError = -1;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java
index b8bfae41b2314..96563d917bf8f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java
@@ -18,14 +18,15 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.DoubleArray;
@@ -190,7 +191,11 @@ static Function<byte[], Number> getPointReaderOrNull(SearchContext context, Aggr
                 converter = ((NumberFieldMapper.NumberFieldType) fieldType)::parsePoint;
             } else if (fieldType.getClass() == DateFieldMapper.DateFieldType.class) {
                 DateFieldMapper.DateFieldType dft = (DateFieldMapper.DateFieldType) fieldType;
-                converter = dft.resolution()::parsePointAsMillis;
+                /*
+                 * Makes sure that nanoseconds decode to milliseconds, just
+                 * like they do when you run the agg without the optimization.
+                 */
+                converter = (in) -> dft.resolution().toInstant(LongPoint.decodeDimension(in, 0)).toEpochMilli();
             }
             return converter;
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
index ba7c49cee7eb4..a131dde9c7673 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
@@ -24,10 +24,16 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 public abstract class SiblingPipelineAggregator extends PipelineAggregator {
     protected SiblingPipelineAggregator(String name, String[] bucketsPaths, Map<String, Object> metaData) {
@@ -41,13 +47,39 @@ protected SiblingPipelineAggregator(String name, String[] bucketsPaths, Map<Stri
         super(in);
     }
 
+    @SuppressWarnings("unchecked")
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
-        return aggregation.copyWithRewritenBuckets(aggregations -> {
-            List<InternalAggregation> aggs = aggregations.copyResults();
-            aggs.add(doReduce(aggregations, reduceContext));
-            return new InternalAggregations(aggs);
-        });
+        if (aggregation instanceof InternalMultiBucketAggregation) {
+            @SuppressWarnings("rawtypes")
+            InternalMultiBucketAggregation multiBucketsAgg = (InternalMultiBucketAggregation) aggregation;
+            List<? extends Bucket> buckets = multiBucketsAgg.getBuckets();
+            List<Bucket> newBuckets = new ArrayList<>();
+            for (Bucket bucket1 : buckets) {
+                InternalMultiBucketAggregation.InternalBucket bucket = (InternalMultiBucketAggregation.InternalBucket) bucket1;
+                InternalAggregation aggToAdd = doReduce(bucket.getAggregations(), reduceContext);
+                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false)
+                    .map((p) -> (InternalAggregation) p)
+                    .collect(Collectors.toList());
+                aggs.add(aggToAdd);
+                InternalMultiBucketAggregation.InternalBucket newBucket = multiBucketsAgg.createBucket(new InternalAggregations(aggs),
+                    bucket);
+                newBuckets.add(newBucket);
+            }
+
+            return multiBucketsAgg.create(newBuckets);
+        } else if (aggregation instanceof InternalSingleBucketAggregation) {
+            InternalSingleBucketAggregation singleBucketAgg = (InternalSingleBucketAggregation) aggregation;
+            InternalAggregation aggToAdd = doReduce(singleBucketAgg.getAggregations(), reduceContext);
+            List<InternalAggregation> aggs = StreamSupport.stream(singleBucketAgg.getAggregations().spliterator(), false)
+                .map((p) -> (InternalAggregation) p)
+                .collect(Collectors.toList());
+            aggs.add(aggToAdd);
+            return singleBucketAgg.create(new InternalAggregations(aggs));
+        } else {
+            throw new IllegalStateException("Aggregation [" + aggregation.getName() + "] must be a bucket aggregation ["
+                    + aggregation.getWriteableName() + "]");
+        }
     }
 
     public abstract InternalAggregation doReduce(Aggregations aggregations, ReduceContext context);
diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
index 5f931d661674e..4d245e2bcb002 100644
--- a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
 
 import java.io.IOException;
 
@@ -45,7 +44,7 @@ public class DfsSearchResult extends SearchPhaseResult {
 
     public DfsSearchResult(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        requestId = in.readLong();
         int termsSize = in.readVInt();
         if (termsSize == 0) {
             terms = EMPTY_TERMS;
@@ -61,9 +60,9 @@ public DfsSearchResult(StreamInput in) throws IOException {
         maxDoc = in.readVInt();
     }
 
-    public DfsSearchResult(SearchContextId contextId, SearchShardTarget shardTarget) {
+    public DfsSearchResult(long id, SearchShardTarget shardTarget) {
         this.setSearchShardTarget(shardTarget);
-        this.contextId = contextId;
+        this.requestId = id;
     }
 
     public DfsSearchResult maxDoc(int maxDoc) {
@@ -100,7 +99,7 @@ public ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics() {
 
   @Override
     public void writeTo(StreamOutput out) throws IOException {
-        contextId.writeTo(out);
+        out.writeLong(requestId);
         out.writeVInt(terms.length);
         for (Term term : terms) {
             out.writeString(term.field());
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
index 81a2f5491a9a3..b213d2bc19d5a 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
@@ -41,12 +40,12 @@ public FetchSearchResult() {
 
     public FetchSearchResult(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        requestId = in.readLong();
         hits = new SearchHits(in);
     }
 
-    public FetchSearchResult(SearchContextId id, SearchShardTarget shardTarget) {
-        this.contextId = id;
+    public FetchSearchResult(long id, SearchShardTarget shardTarget) {
+        this.requestId = id;
         setSearchShardTarget(shardTarget);
     }
 
@@ -87,7 +86,7 @@ public int counterGetAndIncrement() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        contextId.writeTo(out);
+        out.writeLong(requestId);
         hits.writeTo(out);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
index dc76fdf46f792..754735c0cd224 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
@@ -45,8 +44,8 @@ public QueryFetchSearchResult(QuerySearchResult queryResult, FetchSearchResult f
     }
 
     @Override
-    public SearchContextId getContextId() {
-        return queryResult.getContextId();
+    public long getRequestId() {
+        return queryResult.getRequestId();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
index 528d584dfe044..2e7c59b329874 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -40,7 +39,7 @@
  */
 public class ShardFetchRequest extends TransportRequest {
 
-    private SearchContextId contextId;
+    private long id;
 
     private int[] docIds;
 
@@ -51,8 +50,8 @@ public class ShardFetchRequest extends TransportRequest {
     public ShardFetchRequest() {
     }
 
-    public ShardFetchRequest(SearchContextId contextId, IntArrayList list, ScoreDoc lastEmittedDoc) {
-        this.contextId = contextId;
+    public ShardFetchRequest(long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
+        this.id = id;
         this.docIds = list.buffer;
         this.size = list.size();
         this.lastEmittedDoc = lastEmittedDoc;
@@ -60,7 +59,7 @@ public ShardFetchRequest(SearchContextId contextId, IntArrayList list, ScoreDoc
 
     public ShardFetchRequest(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        id = in.readLong();
         size = in.readVInt();
         docIds = new int[size];
         for (int i = 0; i < size; i++) {
@@ -79,7 +78,7 @@ public ShardFetchRequest(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        contextId.writeTo(out);
+        out.writeLong(id);
         out.writeVInt(size);
         for (int i = 0; i < size; i++) {
             out.writeVInt(docIds[i]);
@@ -95,8 +94,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    public SearchContextId contextId() {
-        return contextId;
+    public long id() {
+        return id;
     }
 
     public int[] docIds() {
@@ -118,7 +117,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
 
     @Override
     public String getDescription() {
-        return "id[" + contextId + "], size[" + size + "], lastEmittedDoc[" + lastEmittedDoc + "]";
+        return "id[" + id + "], size[" + size + "], lastEmittedDoc[" + lastEmittedDoc + "]";
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index 813ffc9483375..647d629a390c6 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.search.internal.SearchContextId;
 
 import java.io.IOException;
 
@@ -42,7 +41,7 @@ public ShardFetchSearchRequest() {
 
     }
 
-    public ShardFetchSearchRequest(OriginalIndices originalIndices, SearchContextId id, IntArrayList list, ScoreDoc lastEmittedDoc) {
+    public ShardFetchSearchRequest(OriginalIndices originalIndices, long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
         super(id, list, lastEmittedDoc);
         this.originalIndices = originalIndices;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
index d1743e6182e2f..b792be60f78fc 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
@@ -103,12 +102,6 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         ArrayList<TextFragment> fragsList = new ArrayList<>();
         List<Object> textsToHighlight;
         Analyzer analyzer = context.getMapperService().documentMapper().mappers().indexAnalyzer();
-        Integer keywordIgnoreAbove = null;
-        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
-            KeywordFieldMapper mapper = (KeywordFieldMapper) context.getMapperService().documentMapper()
-                .mappers().getMapper(highlighterContext.fieldName);
-            keywordIgnoreAbove = mapper.ignoreAbove();
-        }
         final int maxAnalyzedOffset = context.getIndexSettings().getHighlightMaxAnalyzedOffset();
 
         try {
@@ -117,11 +110,7 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
 
             for (Object textToHighlight : textsToHighlight) {
                 String text = convertFieldValue(fieldType, textToHighlight);
-                int textLength = text.length();
-                if (keywordIgnoreAbove != null  && textLength > keywordIgnoreAbove) {
-                    continue; // skip highlighting keyword terms that were ignored during indexing
-                }
-                if (textLength > maxAnalyzedOffset) {
+                if (text.length() > maxAnalyzedOffset) {
                     throw new IllegalArgumentException(
                         "The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +
                             "] doc of [" + context.index().getName() + "] index " +
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
index 28ff7220d9bd7..81f700920f827 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
@@ -66,16 +65,11 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
         Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
         final int maxAnalyzedOffset = context.getIndexSettings().getHighlightMaxAnalyzedOffset();
-        Integer keywordIgnoreAbove = null;
-        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
-            KeywordFieldMapper mapper = (KeywordFieldMapper) context.getMapperService().documentMapper()
-                .mappers().getMapper(highlighterContext.fieldName);
-            keywordIgnoreAbove = mapper.ignoreAbove();
-        }
 
         List<Snippet> snippets = new ArrayList<>();
         int numberOfFragments = field.fieldOptions().numberOfFragments();
         try {
+
             final Analyzer analyzer = getAnalyzer(context.getMapperService().documentMapper(), hitContext);
             List<Object> fieldValues = loadFieldValues(fieldType, field, context, hitContext,
                 highlighterContext.highlight.forceSource(field));
@@ -87,11 +81,7 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
             final CustomUnifiedHighlighter highlighter;
             final String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
             final OffsetSource offsetSource = getOffsetSource(fieldType);
-            int fieldValueLength = fieldValue.length();
-            if (keywordIgnoreAbove != null  && fieldValueLength > keywordIgnoreAbove) {
-                return null; // skip highlighting keyword terms that were ignored during indexing
-            }
-            if ((offsetSource == OffsetSource.ANALYSIS) && (fieldValueLength > maxAnalyzedOffset)) {
+            if ((offsetSource == OffsetSource.ANALYSIS) && (fieldValue.length() > maxAnalyzedOffset)) {
                 throw new IllegalArgumentException(
                     "The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +
                         "] doc of [" + context.index().getName() + "] index " + "has exceeded [" +
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 41c46fc6a81be..8c04954a4efcf 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -106,7 +106,7 @@ public Query buildFilteredQuery(Query query) {
     }
 
     @Override
-    public SearchContextId id() {
+    public long id() {
         return in.id();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
index ff86e44c1704b..fe8d173711c69 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
@@ -33,33 +33,33 @@
 
 public class InternalScrollSearchRequest extends TransportRequest {
 
-    private SearchContextId contextId;
+    private long id;
 
     private Scroll scroll;
 
     public InternalScrollSearchRequest() {
     }
 
-    public InternalScrollSearchRequest(SearchScrollRequest request, SearchContextId contextId) {
-        this.contextId = contextId;
+    public InternalScrollSearchRequest(SearchScrollRequest request, long id) {
+        this.id = id;
         this.scroll = request.scroll();
     }
 
     public InternalScrollSearchRequest(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        id = in.readLong();
         scroll = in.readOptionalWriteable(Scroll::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        contextId.writeTo(out);
+        out.writeLong(id);
         out.writeOptionalWriteable(scroll);
     }
 
-    public SearchContextId contextId() {
-        return contextId;
+    public long id() {
+        return id;
     }
 
     public Scroll scroll() {
@@ -78,7 +78,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
 
     @Override
     public String getDescription() {
-        return "id[" + contextId.getId() + "], scroll[" + scroll + "]";
+        return "id[" + id + "], scroll[" + scroll + "]";
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 67577d61a7454..4473546f76d15 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -132,7 +132,7 @@ protected void alreadyClosed() {
      *  alias filters, types filters, etc. */
     public abstract Query buildFilteredQuery(Query query);
 
-    public abstract SearchContextId id();
+    public abstract long id();
 
     public abstract String source();
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
deleted file mode 100644
index 38513dcc5b7d3..0000000000000
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-
-import java.io.IOException;
-import java.util.Objects;
-
-
-public final class SearchContextId implements Writeable {
-    private final String readerId;
-    private final long id;
-
-    public SearchContextId(String readerId, long id) {
-        this.readerId = Objects.requireNonNull(readerId);
-        this.id = id;
-    }
-
-    public SearchContextId(StreamInput in) throws IOException {
-        this.id = in.readLong();
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-            this.readerId = in.readString();
-        } else {
-            this.readerId = "";
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeLong(id);
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-            out.writeString(readerId);
-        }
-    }
-
-    public String getReaderId() {
-        return readerId;
-    }
-
-    public long getId() {
-        return id;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        SearchContextId other = (SearchContextId) o;
-        return id == other.id && readerId.equals(other.readerId);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(readerId, id);
-    }
-
-    @Override
-    public String toString() {
-        return "[" + readerId + "][" + id + "]";
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
index d85d6e674c634..d3919ec3aba48 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -37,21 +36,24 @@
 
 public class QuerySearchRequest extends TransportRequest implements IndicesRequest {
 
-    private final SearchContextId contextId;
+    private long id;
 
-    private final AggregatedDfs dfs;
+    private AggregatedDfs dfs;
 
-    private final OriginalIndices originalIndices;
+    private OriginalIndices originalIndices;
 
-    public QuerySearchRequest(OriginalIndices originalIndices, SearchContextId contextId, AggregatedDfs dfs) {
-        this.contextId = contextId;
+    public QuerySearchRequest() {
+    }
+
+    public QuerySearchRequest(OriginalIndices originalIndices, long id, AggregatedDfs dfs) {
+        this.id = id;
         this.dfs = dfs;
         this.originalIndices = originalIndices;
     }
 
     public QuerySearchRequest(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        id = in.readLong();
         dfs = new AggregatedDfs(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
     }
@@ -59,13 +61,13 @@ public QuerySearchRequest(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        contextId.writeTo(out);
+        out.writeLong(id);
         dfs.writeTo(out);
         OriginalIndices.writeOriginalIndices(originalIndices, out);
     }
 
-    public SearchContextId contextId() {
-        return contextId;
+    public long id() {
+        return id;
     }
 
     public AggregatedDfs dfs() {
@@ -90,7 +92,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
     public String getDescription() {
         StringBuilder sb = new StringBuilder();
         sb.append("id[");
-        sb.append(contextId);
+        sb.append(id);
         sb.append("], ");
         sb.append("indices[");
         Strings.arrayToDelimitedString(originalIndices.indices(), ",", sb);
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 18af04add2082..8c62d229fe766 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 
@@ -78,13 +77,13 @@ public QuerySearchResult(StreamInput in) throws IOException {
             isNull = false;
         }
         if (isNull == false) {
-            SearchContextId id = new SearchContextId(in);
+            long id = in.readLong();
             readFromWithId(id, in);
         }
     }
 
-    public QuerySearchResult(SearchContextId id, SearchShardTarget shardTarget) {
-        this.contextId = id;
+    public QuerySearchResult(long id, SearchShardTarget shardTarget) {
+        this.requestId = id;
         setSearchShardTarget(shardTarget);
         isNull = false;
     }
@@ -299,8 +298,8 @@ public boolean hasSearchContext() {
         return hasScoreDocs || hasSuggestHits();
     }
 
-    public void readFromWithId(SearchContextId id, StreamInput in) throws IOException {
-        this.contextId = id;
+    public void readFromWithId(long id, StreamInput in) throws IOException {
+        this.requestId = id;
         from = in.readVInt();
         size = in.readVInt();
         int numSortFieldsPlus1 = in.readVInt();
@@ -345,7 +344,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(isNull);
         }
         if (isNull == false) {
-            contextId.writeTo(out);
+            out.writeLong(requestId);
             writeToNoId(out);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
index 703ae6939aa8f..ab02d6df7993b 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
@@ -55,14 +55,14 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Return the minimum value.
      */
-    public T getMin() {
+    T getMin() {
         return minValue;
     }
 
     /**
      * Return the maximum value.
      */
-    public T getMax() {
+    T getMax() {
         return maxValue;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortValue.java b/server/src/main/java/org/elasticsearch/search/sort/SortValue.java
index 92fbe3cfc4875..f4cfb812b7a6d 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/SortValue.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/SortValue.java
@@ -117,11 +117,6 @@ public final XContentBuilder toXContent(XContentBuilder builder, DocValueFormat
     @Override
     public abstract String toString();
 
-    /**
-     * Return this {@linkplain SortValue} as a boxed {@linkplain Number}.
-     */
-    public abstract Number numberValue();
-
     private static class DoubleSortValue extends SortValue {
         public static final String NAME = "double";
 
@@ -184,11 +179,6 @@ public int hashCode() {
         public String toString() {
             return Double.toString(key);
         }
-
-        @Override
-        public Number numberValue() {
-            return key;
-        }
     }
 
     private static class LongSortValue extends SortValue {
@@ -253,10 +243,5 @@ public int hashCode() {
         public String toString() {
             return Long.toString(key);
         }
-
-        @Override
-        public Number numberValue() {
-            return key;
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/tasks/Task.java b/server/src/main/java/org/elasticsearch/tasks/Task.java
index 0c821d030e624..1f89a7d88b993 100644
--- a/server/src/main/java/org/elasticsearch/tasks/Task.java
+++ b/server/src/main/java/org/elasticsearch/tasks/Task.java
@@ -140,13 +140,6 @@ public long getStartTime() {
         return startTime;
     }
 
-    /**
-     * Returns the task's start time in nanoseconds ({@link System#nanoTime()} style).
-     */
-    public long getStartTimeNanos() {
-        return startTimeNanos;
-    }
-
     /**
      * Returns id of the parent task or NO_PARENT_ID if the task doesn't have any parent tasks
      */
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 5a2b83c67222d..fe4944fce06a8 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -68,6 +68,7 @@ public class ThreadPool implements Scheduler {
     public static class Names {
         public static final String SAME = "same";
         public static final String GENERIC = "generic";
+        public static final String LISTENER = "listener";
         public static final String GET = "get";
         public static final String ANALYZE = "analyze";
         public static final String WRITE = "write";
@@ -114,6 +115,7 @@ public static ThreadPoolType fromType(String type) {
     public static final Map<String, ThreadPoolType> THREAD_POOL_TYPES = Map.ofEntries(
         entry(Names.SAME, ThreadPoolType.DIRECT),
         entry(Names.GENERIC, ThreadPoolType.SCALING),
+        entry(Names.LISTENER, ThreadPoolType.FIXED),
         entry(Names.GET, ThreadPoolType.FIXED),
         entry(Names.ANALYZE, ThreadPoolType.FIXED),
         entry(Names.WRITE, ThreadPoolType.FIXED),
@@ -170,6 +172,7 @@ public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBui
         builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5)));
         // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded
         // the assumption here is that the listeners should be very lightweight on the listeners side
+        builders.put(Names.LISTENER, new FixedExecutorBuilder(settings, Names.LISTENER, halfProcMaxAt10, -1, false));
         builders.put(Names.FLUSH, new ScalingExecutorBuilder(Names.FLUSH, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
         builders.put(Names.REFRESH, new ScalingExecutorBuilder(Names.REFRESH, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5)));
         builders.put(Names.WARMER, new ScalingExecutorBuilder(Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index 2d314a591b57a..5ce6de7998801 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -272,9 +272,9 @@ private ProxyModeInfo(StreamInput input) throws IOException {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field("proxy_address", address);
-            builder.field("num_proxy_sockets_connected", numSocketsConnected);
-            builder.field("max_proxy_socket_connections", maxSocketConnections);
+            builder.field("address", address);
+            builder.field("num_sockets_connected", numSocketsConnected);
+            builder.field("max_socket_connections", maxSocketConnections);
             return builder;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
index 0321f247b0843..52c1bd7a24d24 100644
--- a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
+++ b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
@@ -57,7 +56,6 @@
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.RemoteTransportException;
 
@@ -820,8 +818,7 @@ public void testFailureToAndFromXContentWithDetails() throws IOException {
                                 new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 61), null,
                                     OriginalIndices.NONE)), new ShardSearchFailure(new RepositoryException("repository_g", "Repo"),
                                 new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62), null,
-                                    OriginalIndices.NONE)), new ShardSearchFailure(
-                                        new SearchContextMissingException(new SearchContextId(UUIDs.randomBase64UUID(), 0L)), null)
+                                    OriginalIndices.NONE)), new ShardSearchFailure(new SearchContextMissingException(0L), null)
                 };
                 failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
 
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index f2a73573ee1b7..cad156555417e 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -79,7 +79,6 @@
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -121,7 +120,6 @@
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.test.TestSearchContext.SHARD_TARGET;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class ExceptionSerializationTests extends ESTestCase {
@@ -353,15 +351,9 @@ public void testActionTransportException() throws IOException {
     }
 
     public void testSearchContextMissingException() throws IOException {
-        SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomLong());
-        Version version = VersionUtils.randomVersion(random());
-        SearchContextMissingException ex = serialize(new SearchContextMissingException(contextId), version);
-        assertThat(ex.contextId().getId(), equalTo(contextId.getId()));
-        if (version.onOrAfter(Version.V_7_7_0)) {
-            assertThat(ex.contextId().getReaderId(), equalTo(contextId.getReaderId()));
-        } else {
-            assertThat(ex.contextId().getReaderId(), equalTo(""));
-        }
+        long id = randomLong();
+        SearchContextMissingException ex = serialize(new SearchContextMissingException(id));
+        assertEquals(id, ex.id());
     }
 
     public void testCircuitBreakingException() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java
deleted file mode 100644
index edf334155238b..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.cluster.node.stats;
-
-import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class NodesStatsRequestTests extends ESTestCase {
-
-    /**
-     * Make sure that we can set, serialize, and deserialize arbitrary sets
-     * of metrics.
-     *
-     * TODO: Use a looping construct rather than direct API calls
-     */
-    public void testMetricsSetters() throws Exception {
-        NodesStatsRequest request = new NodesStatsRequest(randomAlphaOfLength(8));
-        request.indices(randomFrom(CommonStatsFlags.ALL));
-        request.os(randomBoolean());
-        request.process(randomBoolean());
-        request.jvm(randomBoolean());
-        request.threadPool(randomBoolean());
-        request.fs(randomBoolean());
-        request.transport(randomBoolean());
-        request.http(randomBoolean());
-        request.breaker(randomBoolean());
-        request.script(randomBoolean());
-        request.discovery(randomBoolean());
-        request.ingest(randomBoolean());
-        request.adaptiveSelection(randomBoolean());
-        NodesStatsRequest deserializedRequest = roundTripRequest(request);
-        assertRequestsEqual(request, deserializedRequest);
-    }
-
-    /**
-     * Test that a newly constructed NodesStatsRequestObject requests all of the
-     * possible metrics defined in {@link NodesStatsRequest}.
-     */
-    public void testNodesStatsRequestDefaults() {
-        NodesStatsRequest defaultNodesStatsRequest = new NodesStatsRequest(randomAlphaOfLength(8));
-        NodesStatsRequest constructedNodesStatsRequest = new NodesStatsRequest(randomAlphaOfLength(8));
-        constructedNodesStatsRequest.clear();
-        constructedNodesStatsRequest.indices(CommonStatsFlags.ALL);
-
-        assertRequestsEqual(defaultNodesStatsRequest, constructedNodesStatsRequest);
-    }
-
-    /**
-     * Test that the {@link NodesStatsRequest#all()} method sets all of the
-     * metrics to {@code true}.
-     *
-     * TODO: Use a looping construct rather than direct API calls
-     */
-    public void testNodesInfoRequestAll() throws Exception {
-        NodesStatsRequest request = new NodesStatsRequest("node");
-        request.all();
-
-        assertThat(request.indices().getFlags(), equalTo(CommonStatsFlags.ALL.getFlags()));
-        assertTrue(request.os());
-        assertTrue(request.process());
-        assertTrue(request.jvm());
-        assertTrue(request.threadPool());
-        assertTrue(request.fs());
-        assertTrue(request.transport());
-        assertTrue(request.http());
-        assertTrue(request.breaker());
-        assertTrue(request.script());
-        assertTrue(request.discovery());
-        assertTrue(request.ingest());
-        assertTrue(request.adaptiveSelection());
-    }
-
-    /**
-     * Test that the {@link NodesStatsRequest#clear()} method sets all of the
-     * metrics to {@code false}.
-     *
-     * TODO: Use a looping construct rather than direct API calls
-     */
-    public void testNodesInfoRequestClear() throws Exception {
-        NodesStatsRequest request = new NodesStatsRequest("node");
-        request.clear();
-
-        assertThat(request.indices().getFlags(), equalTo(CommonStatsFlags.NONE.getFlags()));
-        assertFalse(request.os());
-        assertFalse(request.process());
-        assertFalse(request.jvm());
-        assertFalse(request.threadPool());
-        assertFalse(request.fs());
-        assertFalse(request.transport());
-        assertFalse(request.http());
-        assertFalse(request.breaker());
-        assertFalse(request.script());
-        assertFalse(request.discovery());
-        assertFalse(request.ingest());
-        assertFalse(request.adaptiveSelection());
-    }
-
-    /**
-     * Serialize and deserialize a request.
-     * @param request A request to serialize.
-     * @return The deserialized, "round-tripped" request.
-     */
-    private static NodesStatsRequest roundTripRequest(NodesStatsRequest request) throws Exception {
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            request.writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                return new NodesStatsRequest(in);
-            }
-        }
-    }
-
-    private static void assertRequestsEqual(NodesStatsRequest request1, NodesStatsRequest request2) {
-        // TODO: Use a looping construct rather than direct API calls
-        assertThat(request1.indices().getFlags(), equalTo(request2.indices().getFlags()));
-        assertThat(request1.os(), equalTo(request2.os()));
-        assertThat(request1.process(), equalTo(request2.process()));
-        assertThat(request1.jvm(), equalTo(request2.jvm()));
-        assertThat(request1.threadPool(), equalTo(request2.threadPool()));
-        assertThat(request1.fs(), equalTo(request2.fs()));
-        assertThat(request1.transport(), equalTo(request2.transport()));
-        assertThat(request1.http(), equalTo(request2.http()));
-        assertThat(request1.breaker(), equalTo(request2.breaker()));
-        assertThat(request1.script(), equalTo(request2.script()));
-        assertThat(request1.discovery(), equalTo(request2.discovery()));
-        assertThat(request1.ingest(), equalTo(request2.ingest()));
-        assertThat(request1.adaptiveSelection(), equalTo(request2.adaptiveSelection()));
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
index 2254c6ba1032f..59518b6e9966f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
@@ -61,6 +61,16 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) {
         instance.partial(randomBoolean());
         instance.includeAliases(randomBoolean());
 
+        if (randomBoolean()) {
+            Map<String, Object> settings = new HashMap<>();
+            int count = randomInt(3) + 1;
+
+            for (int i = 0; i < count; ++i) {
+                settings.put(randomAlphaOfLengthBetween(2, 5), randomAlphaOfLengthBetween(2, 5));
+            }
+
+            instance.settings(settings);
+        }
         if (randomBoolean()) {
             Map<String, Object> indexSettings = new HashMap<>();
             int count = randomInt(3) + 1;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java
deleted file mode 100644
index 095a166f4f0b7..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.indices.forcemerge;
-
-import org.elasticsearch.action.admin.indices.flush.FlushResponse;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class ForceMergeIT extends ESIntegTestCase {
-
-    public void testForceMergeUUIDConsistent() throws IOException {
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        final String index = "test-index";
-        createIndex(index,
-            Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build());
-        ensureGreen(index);
-        final ClusterState state = clusterService().state();
-        final IndexRoutingTable indexShardRoutingTables = state.routingTable().getIndicesRouting().get(index);
-        final IndexShardRoutingTable shardRouting = indexShardRoutingTables.getShards().get(0);
-        final String primaryNodeId = shardRouting.primaryShard().currentNodeId();
-        final String replicaNodeId = shardRouting.replicaShards().get(0).currentNodeId();
-        final Index idx = shardRouting.primaryShard().index();
-        final IndicesService primaryIndicesService =
-            internalCluster().getInstance(IndicesService.class, state.nodes().get(primaryNodeId).getName());
-        final IndicesService replicaIndicesService = internalCluster().getInstance(IndicesService.class,
-            state.nodes().get(replicaNodeId).getName());
-        final IndexShard primary = primaryIndicesService.indexService(idx).getShard(0);
-        final IndexShard replica = replicaIndicesService.indexService(idx).getShard(0);
-
-        assertThat(getForceMergeUUID(primary), nullValue());
-        assertThat(getForceMergeUUID(replica), nullValue());
-
-        final ForceMergeResponse forceMergeResponse =
-            client().admin().indices().prepareForceMerge(index).setMaxNumSegments(1).get();
-
-        assertThat(forceMergeResponse.getFailedShards(), is(0));
-        assertThat(forceMergeResponse.getSuccessfulShards(), is(2));
-
-        // Force flush to force a new commit that contains the force flush UUID
-        final FlushResponse flushResponse = client().admin().indices().prepareFlush(index).setForce(true).get();
-        assertThat(flushResponse.getFailedShards(), is(0));
-        assertThat(flushResponse.getSuccessfulShards(), is(2));
-
-        final String primaryForceMergeUUID = getForceMergeUUID(primary);
-        assertThat(primaryForceMergeUUID, notNullValue());
-
-        final String replicaForceMergeUUID = getForceMergeUUID(replica);
-        assertThat(replicaForceMergeUUID, notNullValue());
-        assertThat(primaryForceMergeUUID, is(replicaForceMergeUUID));
-    }
-
-    private static String getForceMergeUUID(IndexShard indexShard) throws IOException {
-        try (Engine.IndexCommitRef indexCommitRef = indexShard.acquireLastIndexCommit(true)) {
-            return indexCommitRef.getIndexCommit().getUserData().get(Engine.FORCE_MERGE_UUID_KEY);
-        }
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index bd45128160e56..a01c1f44a73d9 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.ingest.CompoundProcessor;
 import org.elasticsearch.ingest.IngestDocument;
@@ -46,7 +45,6 @@
 import static org.elasticsearch.ingest.IngestDocument.MetaData.ROUTING;
 import static org.elasticsearch.ingest.IngestDocument.MetaData.VERSION;
 import static org.elasticsearch.ingest.IngestDocument.MetaData.VERSION_TYPE;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -218,33 +216,4 @@ public void testNonExistentPipelineId() {
             () -> SimulatePipelineRequest.parseWithPipelineId(pipelineId, requestContent, false, ingestService));
         assertThat(e.getMessage(), equalTo("pipeline [" + pipelineId + "] does not exist"));
     }
-
-    public void testNotValidDocs() {
-        Map<String, Object> requestContent = new HashMap<>();
-        List<Map<String, Object>> docs = new ArrayList<>();
-        Map<String, Object> pipelineConfig = new HashMap<>();
-        List<Map<String, Object>> processors = new ArrayList<>();
-        pipelineConfig.put("processors", processors);
-        requestContent.put(Fields.DOCS, docs);
-        requestContent.put(Fields.PIPELINE, pipelineConfig);
-        Exception e1 = expectThrows(IllegalArgumentException.class,
-            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService));
-        assertThat(e1.getMessage(), equalTo("must specify at least one document in [docs]"));
-
-        List<String> stringList = new ArrayList<>();
-        stringList.add("test");
-        pipelineConfig.put("processors", processors);
-        requestContent.put(Fields.DOCS, stringList);
-        requestContent.put(Fields.PIPELINE, pipelineConfig);
-        Exception e2 = expectThrows(IllegalArgumentException.class,
-            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService));
-        assertThat(e2.getMessage(), equalTo("malformed [docs] section, should include an inner object"));
-
-        docs.add(new HashMap<>());
-        requestContent.put(Fields.DOCS, docs);
-        requestContent.put(Fields.PIPELINE, pipelineConfig);
-        Exception e3 = expectThrows(ElasticsearchParseException.class,
-            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService));
-        assertThat(e3.getMessage(), containsString("required property is missing"));
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
index b8957b7fef272..2583e1a76c427 100644
--- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -22,10 +22,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
@@ -35,7 +33,6 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
@@ -58,7 +55,7 @@
 public class AbstractSearchAsyncActionTests extends ESTestCase {
 
     private final List<Tuple<String, String>> resolvedNodes = new ArrayList<>();
-    private final Set<SearchContextId> releasedContexts = new CopyOnWriteArraySet<>();
+    private final Set<Long> releasedContexts = new CopyOnWriteArraySet<>();
 
     private AbstractSearchAsyncAction<SearchPhaseResult> createAction(SearchRequest request,
                                                                       ArraySearchPhaseResults<SearchPhaseResult> results,
@@ -93,7 +90,7 @@ private AbstractSearchAsyncAction<SearchPhaseResult> createAction(SearchRequest
                     Collections.singletonList(
                         new SearchShardIterator(null, null, Collections.emptyList(), null)
                     )
-                ), timeProvider, ClusterState.EMPTY_STATE, null,
+                ), timeProvider, 0, null,
                 results, request.getMaxConcurrentShardRequests(),
                 SearchResponse.Clusters.EMPTY) {
             @Override
@@ -113,8 +110,7 @@ long buildTookInMillis() {
             }
 
             @Override
-            public void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection,
-                                                 OriginalIndices originalIndices) {
+            public void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
                 releasedContexts.add(contextId);
             }
         };
@@ -195,7 +191,7 @@ public void testSendSearchResponseDisallowPartialFailures() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false);
         AtomicReference<Exception> exception = new AtomicReference<>();
         ActionListener<SearchResponse> listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
-        Set<SearchContextId> requestIds = new HashSet<>();
+        Set<Long> requestIds = new HashSet<>();
         List<Tuple<String, String>> nodeLookups = new ArrayList<>();
         int numFailures = randomIntBetween(1, 5);
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = phaseResults(requestIds, nodeLookups, numFailures);
@@ -223,7 +219,7 @@ public void testOnPhaseFailure() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false);
         AtomicReference<Exception> exception = new AtomicReference<>();
         ActionListener<SearchResponse> listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
-        Set<SearchContextId> requestIds = new HashSet<>();
+        Set<Long> requestIds = new HashSet<>();
         List<Tuple<String, String>> nodeLookups = new ArrayList<>();
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = phaseResults(requestIds, nodeLookups, 0);
         AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest, phaseResults, listener, false, new AtomicLong());
@@ -266,16 +262,16 @@ public void testShardNotAvailableWithDisallowPartialFailures() {
         assertEquals(0, searchPhaseExecutionException.getSuppressed().length);
     }
 
-    private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<SearchContextId> contextIds,
-                                                                           List<Tuple<String, String>> nodeLookups,
-                                                                           int numFailures) {
+    private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<Long> requestIds,
+                                                                                              List<Tuple<String, String>> nodeLookups,
+                                                                                              int numFailures) {
         int numResults = randomIntBetween(1, 10);
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = new ArraySearchPhaseResults<>(numResults + numFailures);
 
         for (int i = 0; i < numResults; i++) {
-            SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomNonNegativeLong());
-            contextIds.add(contextId);
-            SearchPhaseResult phaseResult = new PhaseResult(contextId);
+            long requestId = randomLong();
+            requestIds.add(requestId);
+            SearchPhaseResult phaseResult = new PhaseResult(requestId);
             String resultClusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10);
             String resultNodeId = randomAlphaOfLengthBetween(5, 10);
             ShardId resultShardId = new ShardId("index", "index-uuid", i);
@@ -288,8 +284,8 @@ private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<Searc
     }
 
     private static final class PhaseResult extends SearchPhaseResult {
-        PhaseResult(SearchContextId contextId) {
-            this.contextId = contextId;
+        PhaseResult(long requestId) {
+            this.requestId = requestId;
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
index 3d46a3e13a434..c38f77207daf8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
@@ -18,11 +18,9 @@
  */
 package org.elasticsearch.action.search;
 
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -55,8 +53,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.IntStream;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class CanMatchPreFilterSearchPhaseTests extends ESTestCase {
 
     public void testFilterShards() throws InterruptedException {
@@ -94,7 +90,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req
             (clusterAlias, node) -> lookup.get(node),
             Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
             Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
-            searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
+            searchRequest, null, shardsIter, timeProvider, 0, null,
             (iter) -> new SearchPhase("test") {
                     @Override
                     public void run() throws IOException {
@@ -162,7 +158,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req
             (clusterAlias, node) -> lookup.get(node),
             Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
             Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
-            searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
+            searchRequest, null, shardsIter, timeProvider, 0, null,
             (iter) -> new SearchPhase("test") {
                 @Override
                 public void run() throws IOException {
@@ -229,7 +225,7 @@ public void sendCanMatch(
             null,
             shardsIter,
             timeProvider,
-            ClusterState.EMPTY_STATE,
+            0,
             null,
             (iter) -> new AbstractSearchAsyncAction<>(
                 "test",
@@ -247,7 +243,7 @@ public void sendCanMatch(
                 responseListener,
                 iter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                ClusterState.EMPTY_STATE,
+                0,
                 null,
                 new ArraySearchPhaseResults<>(iter.size()),
                 randomIntBetween(1, 32),
@@ -329,7 +325,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req
                 (clusterAlias, node) -> lookup.get(node),
                 Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
                 Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
-                searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
+                searchRequest, null, shardsIter, timeProvider, 0, null,
                 (iter) -> new SearchPhase("test") {
                     @Override
                     public void run() {
@@ -353,76 +349,4 @@ public void run() {
             }
         }
     }
-
-    public void testInvalidSortShards() throws InterruptedException {
-        final TransportSearchAction.SearchTimeProvider timeProvider =
-            new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), System::nanoTime);
-
-        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
-        DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
-        DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
-        lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode));
-        lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode));
-
-        for (SortOrder order : SortOrder.values()) {
-            int numShards = randomIntBetween(2, 20);
-            List<ShardId> shardIds = new ArrayList<>();
-            Set<ShardId> shardToSkip = new HashSet<>();
-
-            SearchTransportService searchTransportService = new SearchTransportService(null, null) {
-                @Override
-                public void sendCanMatch(Transport.Connection connection, ShardSearchRequest request, SearchTask task,
-                                         ActionListener<SearchService.CanMatchResponse> listener) {
-                    final MinAndMax<?> minMax;
-                    if (request.shardId().id() == numShards-1) {
-                        minMax = new MinAndMax<>(new BytesRef("bar"), new BytesRef("baz"));
-                    } else {
-                        Long min = randomLong();
-                        Long max = randomLongBetween(min, Long.MAX_VALUE);
-                        minMax = new MinAndMax<>(min, max);
-                    }
-                    boolean canMatch = frequently();
-                    synchronized (shardIds) {
-                        shardIds.add(request.shardId());
-                        if (canMatch == false) {
-                            shardToSkip.add(request.shardId());
-                        }
-                    }
-                    new Thread(() -> listener.onResponse(new SearchService.CanMatchResponse(canMatch, minMax))).start();
-                }
-            };
-
-            AtomicReference<GroupShardsIterator<SearchShardIterator>> result = new AtomicReference<>();
-            CountDownLatch latch = new CountDownLatch(1);
-            GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("logs",
-                new OriginalIndices(new String[]{"logs"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
-                numShards, randomBoolean(), primaryNode, replicaNode);
-            final SearchRequest searchRequest = new SearchRequest();
-            searchRequest.source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp").order(order)));
-            searchRequest.allowPartialSearchResults(true);
-
-            CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase(logger,
-                searchTransportService,
-                (clusterAlias, node) -> lookup.get(node),
-                Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
-                Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
-                searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
-                (iter) -> new SearchPhase("test") {
-                    @Override
-                    public void run() {
-                        result.set(iter);
-                        latch.countDown();
-                    }
-                }, SearchResponse.Clusters.EMPTY);
-
-            canMatchPhase.start();
-            latch.await();
-            int shardId = 0;
-            for (SearchShardIterator i : result.get()) {
-                assertThat(i.shardId().id(), equalTo(shardId++));
-                assertEquals(shardToSkip.contains(i.shardId()), i.skip());
-            }
-            assertThat(result.get().size(), equalTo(numShards));
-        }
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
index 616b382baf5ca..bcb4a1200b7e8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
@@ -23,12 +23,10 @@
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.NodeNotConnectedException;
 import org.elasticsearch.transport.Transport;
@@ -94,20 +92,17 @@ public void testClearScrollIds() throws IOException, InterruptedException {
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 1), node1);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 12), node2);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 42), node3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
         array.setOnce(2, testSearchPhaseResult3);
         AtomicInteger numFreed = new AtomicInteger(0);
-        String scrollId = TransportSearchHelper.buildScrollId(array, randomBoolean());
+        String scrollId = TransportSearchHelper.buildScrollId(array);
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
         CountDownLatch latch = new CountDownLatch(1);
         ActionListener<ClearScrollResponse> listener = new LatchedActionListener<>(new ActionListener<ClearScrollResponse>() {
@@ -126,7 +121,7 @@ public void onFailure(Exception e) {
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
-            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
+            public void sendFreeContext(Transport.Connection connection, long contextId,
                                         ActionListener<SearchFreeContextResponse> listener) {
                 nodesInvoked.add(connection.getNode());
                 boolean freed = randomBoolean();
@@ -158,14 +153,11 @@ public void testClearScrollIdsWithFailure() throws IOException, InterruptedExcep
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 1), node1);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 12), node2);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 42), node3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
@@ -173,7 +165,7 @@ public void testClearScrollIdsWithFailure() throws IOException, InterruptedExcep
         AtomicInteger numFreed = new AtomicInteger(0);
         AtomicInteger numFailures = new AtomicInteger(0);
         AtomicInteger numConnectionFailures = new AtomicInteger(0);
-        String scrollId = TransportSearchHelper.buildScrollId(array, randomBoolean());
+        String scrollId = TransportSearchHelper.buildScrollId(array);
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
         CountDownLatch latch = new CountDownLatch(1);
 
@@ -197,7 +189,7 @@ public void onFailure(Exception e) {
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
-            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
+            public void sendFreeContext(Transport.Connection connection, long contextId,
                                         ActionListener<SearchFreeContextResponse> listener) {
                 nodesInvoked.add(connection.getNode());
                 boolean freed = randomBoolean();
diff --git a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
index 8d20289ca1ae9..a85f63abd0967 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
@@ -19,13 +19,11 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -61,7 +59,7 @@ public void testCollect() throws InterruptedException {
                 case 1:
                     state.add(1);
                     executor.execute(() -> {
-                        DfsSearchResult dfsSearchResult = new DfsSearchResult(new SearchContextId(UUIDs.randomBase64UUID(), shardID), null);
+                        DfsSearchResult dfsSearchResult = new DfsSearchResult(shardID, null);
                         dfsSearchResult.setShardIndex(shardID);
                         dfsSearchResult.setSearchShardTarget(new SearchShardTarget("foo",
                             new ShardId("bar", "baz", shardID), null, OriginalIndices.NONE));
@@ -86,7 +84,7 @@ public void testCollect() throws InterruptedException {
                     break;
                 case 1:
                     assertNotNull(results.get(i));
-                    assertEquals(i, results.get(i).getContextId().getId());
+                    assertEquals(i, results.get(i).getRequestId());
                     break;
                 case 2:
                     final int shardId = i;
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index a2496c19d2a9f..8a8e28e15f20d 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -25,7 +25,6 @@
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -35,7 +34,6 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
@@ -47,8 +45,8 @@
 
 public class DfsQueryPhaseTests extends ESTestCase {
 
-    private static DfsSearchResult newSearchResult(int shardIndex, SearchContextId contextId, SearchShardTarget target) {
-        DfsSearchResult result = new DfsSearchResult(contextId, target);
+    private static DfsSearchResult newSearchResult(int shardIndex, long requestId, SearchShardTarget target) {
+        DfsSearchResult result = new DfsSearchResult(requestId, target);
         result.setShardIndex(shardIndex);
         return result;
     }
@@ -56,10 +54,8 @@ private static DfsSearchResult newSearchResult(int shardIndex, SearchContextId c
     public void testDfsWith2Shards() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, new SearchContextId(UUIDs.randomBase64UUID(), 1),
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, new SearchContextId(UUIDs.randomBase64UUID(), 2),
-            new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -69,24 +65,24 @@ public void testDfsWith2Shards() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.contextId().getId() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
-                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+                if (request.id() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+                        null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.contextId().getId() == 2) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
-                        new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+                } else if (request.id() == 2) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node2", new ShardId("test", "na", 0),
+                        null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F),
                             new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
                 } else {
-                    fail("no such request ID: " + request.contextId());
+                    fail("no such request ID: " + request.id());
                 }
             }
         };
@@ -118,10 +114,8 @@ public void run() throws IOException {
     public void testDfsWith1ShardFailed() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        final SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 1);
-        final SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 2);
-        results.set(0, newSearchResult(0, ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -131,18 +125,18 @@ public void testDfsWith1ShardFailed() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.contextId().getId() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
-                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+                if (request.id() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+                        null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(
                             new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                             new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.contextId().getId() == 2) {
+                } else if (request.id() == 2) {
                     listener.onFailure(new MockDirectoryWrapper.FakeIOException());
                 } else {
-                    fail("no such request ID: " + request.contextId());
+                    fail("no such request ID: " + request.id());
                 }
             }
         };
@@ -169,7 +163,7 @@ public void run() throws IOException {
         assertEquals(1, mockSearchPhaseContext.failures.size());
         assertTrue(mockSearchPhaseContext.failures.get(0).getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx2));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(2L));
         assertNull(responseRef.get().get(1));
     }
 
@@ -177,10 +171,8 @@ public void run() throws IOException {
     public void testFailPhaseOnException() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, new SearchContextId(UUIDs.randomBase64UUID(), 1),
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, new SearchContextId(UUIDs.randomBase64UUID(), 2),
-            new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -190,18 +182,18 @@ public void testFailPhaseOnException() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.contextId().getId() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
-                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+                if (request.id() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+                        null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.contextId().getId() == 2) {
+                } else if (request.id() == 2) {
                    throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
                 } else {
-                    fail("no such request ID: " + request.contextId());
+                    fail("no such request ID: " + request.id());
                 }
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 8978f43842cfe..5eec29dbf8039 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -23,8 +23,6 @@
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.shard.ShardId;
@@ -37,7 +35,6 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
@@ -73,7 +70,7 @@ public void testShortcutQueryAndFetchOptimization() {
             numHits = 0;
         }
 
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
             @Override
             public void run() {
@@ -98,18 +95,15 @@ public void testFetchTwoDocument() {
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        final SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
-        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+            null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        final SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 312);
-        queryResult = new QuerySearchResult(ctx2,
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -121,18 +115,18 @@ public void testFetchTwoDocument() {
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
                 FetchSearchResult fetchResult = new FetchSearchResult();
-                if (request.contextId().equals(ctx2)) {
+                if (request.id() == 321) {
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
-                    assertEquals(ctx1, request.contextId());
+                    assertEquals(123, request.id());
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F));
                 }
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -159,18 +153,15 @@ public void testFailFetchOneDoc() {
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
-        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+            null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 321);
-        queryResult = new QuerySearchResult(ctx2,
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -181,7 +172,7 @@ public void testFailFetchOneDoc() {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
-                if (request.contextId().getId() == 321) {
+                if (request.id() == 321) {
                     FetchSearchResult fetchResult = new FetchSearchResult();
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
@@ -192,7 +183,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
 
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -211,7 +202,7 @@ public void run() {
         assertEquals(1, searchResponse.getShardFailures().length);
         assertTrue(searchResponse.getShardFailures()[0].getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx1));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
     }
 
     public void testFetchDocsConcurrently() throws InterruptedException {
@@ -224,8 +215,8 @@ public void testFetchDocsConcurrently() throws InterruptedException {
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP,
             mockSearchPhaseContext.getRequest(), numHits);
         for (int i = 0; i < numHits; i++) {
-            QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", i),
-                new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+            QuerySearchResult queryResult = new QuerySearchResult(i, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+                null, OriginalIndices.NONE));
             queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(i+1, i)}), i), new DocValueFormat[0]);
             queryResult.size(resultSetSize); // the size of the result set
@@ -238,14 +229,14 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                                          SearchActionListener<FetchSearchResult> listener) {
                 new Thread(() -> {
                     FetchSearchResult fetchResult = new FetchSearchResult();
-                    fetchResult.hits(new SearchHits(new SearchHit[]{new SearchHit((int) (request.contextId().getId() + 1))},
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit((int) (request.id()+1))},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 100F));
                     listener.onResponse(fetchResult);
                 }).start();
             }
         };
         CountDownLatch latch = new CountDownLatch(1);
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -281,16 +272,15 @@ public void testExceptionFailsPhase() {
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+            null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(new SearchContextId("", 321),
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -305,18 +295,18 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 if (numFetches.incrementAndGet() == 1) {
                     throw new RuntimeException("BOOM");
                 }
-                if (request.contextId().getId() == 321) {
+                if (request.id() == 321) {
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
-                    assertEquals(request.contextId().getId(), 123);
+                    assertEquals(request, 123);
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F));
                 }
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -338,18 +328,15 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = 1;
-        SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
-        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+            null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 321);
-        queryResult = new QuerySearchResult(ctx2,
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -361,7 +348,7 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
                 FetchSearchResult fetchResult = new FetchSearchResult();
-                if (request.contextId().equals(ctx2)) {
+                if (request.id() == 321) {
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
@@ -370,7 +357,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -388,6 +375,6 @@ public void run() {
         assertEquals(0, searchResponse.getFailedShards());
         assertEquals(2, searchResponse.getSuccessfulShards());
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx1));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index cd060c971a4f4..28df802fdb409 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.transport.Transport;
 import org.junit.Assert;
@@ -47,7 +46,7 @@ public final class MockSearchPhaseContext implements SearchPhaseContext {
     final AtomicInteger numSuccess;
     final List<ShardSearchFailure> failures = Collections.synchronizedList(new ArrayList<>());
     SearchTransportService searchTransport;
-    final Set<SearchContextId> releasedSearchContexts = new HashSet<>();
+    final Set<Long> releasedSearchContexts = new HashSet<>();
     final SearchRequest searchRequest = new SearchRequest();
     final AtomicReference<SearchResponse> searchResponse = new AtomicReference<>();
 
@@ -136,7 +135,7 @@ public void onFailure(Exception e) {
     }
 
     @Override
-    public void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+    public void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
         releasedSearchContexts.add(contextId);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index 249426057c4ac..bdf2026f43b40 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -21,21 +21,18 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
@@ -109,7 +106,7 @@ public void testSkipSearchShards() throws InterruptedException {
                 responseListener,
                 shardsIter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                ClusterState.EMPTY_STATE,
+                0,
                 null,
                 new ArraySearchPhaseResults<>(shardsIter.size()),
                 request.getMaxConcurrentShardRequests(),
@@ -125,8 +122,7 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
 
                     new Thread(() -> {
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()),
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
                         connection.getNode());
                         listener.onResponse(testSearchPhaseResult);
 
@@ -215,7 +211,7 @@ public void testLimitConcurrentShardRequests() throws InterruptedException {
                 responseListener,
                 shardsIter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                ClusterState.EMPTY_STATE,
+                0,
                 null,
                 new ArraySearchPhaseResults<>(shardsIter.size()),
                 request.getMaxConcurrentShardRequests(),
@@ -236,8 +232,8 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                             throw new AssertionError(e);
                         }
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                            connection.getNode());
                         if (shardFailures[shard.shardId().id()]) {
                             listener.onFailure(new RuntimeException());
                         } else {
@@ -281,7 +277,7 @@ public void testFanOutAndCollect() throws InterruptedException {
         DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
 
-        Map<DiscoveryNode, Set<SearchContextId>> nodeToContextMap = newConcurrentMap();
+        Map<DiscoveryNode, Set<Long>> nodeToContextMap = newConcurrentMap();
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         int numShards = randomIntBetween(1, 10);
         GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
@@ -290,7 +286,7 @@ public void testFanOutAndCollect() throws InterruptedException {
         AtomicInteger numFreedContext = new AtomicInteger();
         SearchTransportService transportService = new SearchTransportService(null, null) {
             @Override
-            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId, OriginalIndices originalIndices) {
+            public void sendFreeContext(Transport.Connection connection, long contextId, OriginalIndices originalIndices) {
                 numFreedContext.incrementAndGet();
                 assertTrue(nodeToContextMap.containsKey(connection.getNode()));
                 assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId));
@@ -318,7 +314,7 @@ public void sendFreeContext(Transport.Connection connection, SearchContextId con
                         responseListener,
                         shardsIter,
                         new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                        ClusterState.EMPTY_STATE,
+                        0,
                         null,
                         new ArraySearchPhaseResults<>(shardsIter.size()),
                         request.getMaxConcurrentShardRequests(),
@@ -330,10 +326,10 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                 listener) {
                 assertTrue("shard: " + shard.shardId() + " has been queried twice", response.queried.add(shard.shardId()));
                 Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                    new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
-                Set<SearchContextId> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
-                ids.add(testSearchPhaseResult.getContextId());
+                TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                    connection.getNode());
+                Set<Long> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
+                ids.add(testSearchPhaseResult.getRequestId());
                 if (randomBoolean()) {
                     listener.onResponse(testSearchPhaseResult);
                 } else {
@@ -349,7 +345,7 @@ public void run() {
                         for (int i = 0; i < results.getNumShards(); i++) {
                             TestSearchPhaseResult result = results.getAtomicArray().get(i);
                             assertEquals(result.node.getId(), result.getSearchShardTarget().getNodeId());
-                            sendReleaseSearchContext(result.getContextId(), new MockConnection(result.node), OriginalIndices.NONE);
+                            sendReleaseSearchContext(result.getRequestId(), new MockConnection(result.node), OriginalIndices.NONE);
                         }
                         responseListener.onResponse(response);
                     }
@@ -423,7 +419,7 @@ public void testAllowPartialResults() throws InterruptedException {
                 responseListener,
                 shardsIter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                ClusterState.EMPTY_STATE,
+                0,
                 null,
                 new ArraySearchPhaseResults<>(shardsIter.size()),
                 request.getMaxConcurrentShardRequests(),
@@ -438,8 +434,8 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                     });
                     new Thread(() -> {
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                            connection.getNode());
                         if (shardIt.remaining() > 0) {
                             numFailReplicas.incrementAndGet();
                             listener.onFailure(new RuntimeException());
@@ -517,8 +513,9 @@ public static class TestSearchResponse extends SearchResponse {
 
     public static class TestSearchPhaseResult extends SearchPhaseResult {
         final DiscoveryNode node;
-        TestSearchPhaseResult(SearchContextId contextId, DiscoveryNode node) {
-            this.contextId = contextId;
+
+        TestSearchPhaseResult(long id, DiscoveryNode node) {
+            this.requestId = id;
             this.node = node;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 5e65487fe5927..f49d3a69caca0 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -31,7 +31,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.text.Text;
@@ -51,7 +50,6 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.Suggest;
@@ -232,7 +230,7 @@ private static AtomicArray<SearchPhaseResult> generateQueryResults(int nShards,
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex),
                 clusterAlias, OriginalIndices.NONE);
-            QuerySearchResult querySearchResult = new QuerySearchResult(new SearchContextId("", shardIndex), searchShardTarget);
+            QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, searchShardTarget);
             final TopDocs topDocs;
             float maxScore = 0;
             if (searchHitsSize == 0) {
@@ -304,7 +302,7 @@ private static AtomicArray<SearchPhaseResult> generateFetchResults(int nShards,
             float maxScore = -1F;
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget shardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), clusterAlias, OriginalIndices.NONE);
-            FetchSearchResult fetchSearchResult = new FetchSearchResult(new SearchContextId("", shardIndex), shardTarget);
+            FetchSearchResult fetchSearchResult = new FetchSearchResult(shardIndex, shardTarget);
             List<SearchHit> searchHits = new ArrayList<>();
             for (ScoreDoc scoreDoc : mergedSearchDocs) {
                 if (scoreDoc.shardIndex == shardIndex) {
@@ -367,8 +365,8 @@ private void consumerTestCase(int numEmptyResponses) {
             numEmptyResponses --;
         }
 
-        QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 0),
-            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
+        QuerySearchResult result = new QuerySearchResult(0, new SearchShardTarget("node", new ShardId("a", "b", 0),
+            null, OriginalIndices.NONE));
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 1.0D, DocValueFormat.RAW,
@@ -377,8 +375,7 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(0);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 1),
-            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
+        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 3.0D, DocValueFormat.RAW,
@@ -387,8 +384,7 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(2);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 1),
-            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
+        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 2.0D, DocValueFormat.RAW,
@@ -456,8 +452,8 @@ public void testConsumerConcurrently() throws InterruptedException {
             threads[i] = new Thread(() -> {
                 int number = randomIntBetween(1, 1000);
                 max.updateAndGet(prev -> Math.max(prev, number));
-                QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), id),
-                    new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE));
+                QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
+                    null, OriginalIndices.NONE));
                 result.topDocs(new TopDocsAndMaxScore(
                     new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, number)}), number),
                     new DocValueFormat[0]);
@@ -500,8 +496,8 @@ public void testConsumerOnlyAggs() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
-                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
+                null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number),
                     new DocValueFormat[0]);
             InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", (double) number,
@@ -538,8 +534,8 @@ public void testConsumerOnlyHits() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
-                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
+                null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(0, number)}), number), new DocValueFormat[0]);
             result.setShardIndex(i);
@@ -605,8 +601,8 @@ public void testReduceTopNWithFromOffset() {
             searchPhaseController.newSearchPhaseResults(NOOP, request, 4);
         int score = 100;
         for (int i = 0; i < 4; i++) {
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
-                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
+                null, OriginalIndices.NONE));
             ScoreDoc[] docs = new ScoreDoc[3];
             for (int j = 0; j < docs.length; j++) {
                 docs[j] = new ScoreDoc(0, score--);
@@ -647,8 +643,8 @@ public void testConsumerSortByField() {
             max.updateAndGet(prev -> Math.max(prev, number));
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, new Object[]{number})};
             TopDocs topDocs = new TopFieldDocs(new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields);
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
-                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
+                null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -685,8 +681,8 @@ public void testConsumerFieldCollapsing() {
             Object[] values = {randomFrom(collapseValues)};
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, values)};
             TopDocs topDocs = new CollapseTopFieldDocs("field", new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields, values);
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
-                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
+                null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -718,8 +714,8 @@ public void testConsumerSuggestions() {
         int maxScorePhrase = -1;
         int maxScoreCompletion = -1;
         for (int i = 0; i < expectedNumResults; i++) {
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
-                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
+                null, OriginalIndices.NONE));
             List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions =
                 new ArrayList<>();
             {
@@ -818,18 +814,18 @@ public void onQueryResult(int shardIndex) {
                 }
 
                 @Override
-                public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
+                public void onQueryFailure(int shardIndex, Exception exc) {
                     assertThat(shardIndex, lessThan(expectedNumResults));
                     numQueryFailureListener.incrementAndGet();
                 }
 
                 @Override
-                public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
-                    assertEquals(numReduceListener.incrementAndGet(), reducePhase);
+                public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {
+                    assertEquals(numReduceListener.incrementAndGet(), version);
                 }
 
                 @Override
-                public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+                public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {
                     totalHitsListener.set(totalHits);
                     finalAggsListener.set(aggs);
                     numReduceListener.incrementAndGet();
@@ -844,8 +840,8 @@ public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, Interna
                 threads[i] = new Thread(() -> {
                     int number = randomIntBetween(1, 1000);
                     max.updateAndGet(prev -> Math.max(prev, number));
-                    QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), id),
-                        new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE));
+                    QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
+                        null, OriginalIndices.NONE));
                     result.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[]{new ScoreDoc(0, number)}), number),
                         new DocValueFormat[0]);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
index 3f9269afd9e78..1a6bb6263fb6b 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
@@ -24,13 +24,11 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.util.ArrayList;
@@ -39,7 +37,6 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -142,8 +139,7 @@ private static void testCase(NodeClient client, SearchRequest request,
         CountDownLatch latch = new CountDownLatch(1);
         SearchProgressActionListener listener = new SearchProgressActionListener() {
             @Override
-            public void onListShards(List<SearchShard> shards, List<SearchShard> skippedShards,
-                                     SearchResponse.Clusters clusters, boolean fetchPhase) {
+            public void onListShards(List<SearchShard> shards, boolean fetchPhase) {
                 shardsListener.set(shards);
                 assertEquals(fetchPhase, hasFetchPhase);
             }
@@ -155,7 +151,7 @@ public void onQueryResult(int shardIndex) {
             }
 
             @Override
-            public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
+            public void onQueryFailure(int shardIndex, Exception exc) {
                 assertThat(shardIndex, lessThan(shardsListener.get().size()));
                 numQueryFailures.incrementAndGet();
             }
@@ -173,12 +169,12 @@ public void onFetchFailure(int shardIndex, Exception exc) {
             }
 
             @Override
-            public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+            public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {
                 numReduces.incrementAndGet();
             }
 
             @Override
-            public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+            public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {
                 numReduces.incrementAndGet();
             }
 
@@ -193,14 +189,7 @@ public void onFailure(Exception e) {
                 throw new AssertionError();
             }
         };
-        client.executeLocally(SearchAction.INSTANCE, new SearchRequest(request) {
-            @Override
-            public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
-                SearchTask task = super.createTask(id, type, action, parentTaskId, headers);
-                task.setProgressListener(listener);
-                return task;
-            }
-        }, listener);
+        client.executeSearchLocally(request, listener);
         latch.await();
 
         assertThat(shardsListener.get(), equalTo(expectedShards));
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
index 49eb7673592bf..2494b45f61174 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
@@ -23,14 +23,12 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 
@@ -50,11 +48,11 @@ public class SearchScrollAsyncActionTests extends ESTestCase {
     public void testSendRequestsToNodes() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId(UUIDs.randomBase64UUID(), 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 0)));
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -75,7 +73,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                 {
                     new Thread(() -> {
                         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
                         testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                             new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                         searchActionListener.onResponse(testSearchPhaseResult);
@@ -111,7 +109,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         ScrollIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
-            assertEquals(context[i].getContextId(), results.get(i).getContextId());
+            assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
             assertEquals(context[i].getNode(), results.get(i).node.getId());
         }
     }
@@ -119,11 +117,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
     public void testFailNextPhase() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("a", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("b", 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("c", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("d", 0)));
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -166,7 +164,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                 {
                     new Thread(() -> {
                         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
                         testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                             new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                         searchActionListener.onResponse(testSearchPhaseResult);
@@ -202,18 +200,18 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         ScrollIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
-            assertEquals(context[i].getContextId(), results.get(i).getContextId());
+            assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
             assertEquals(context[i].getNode(), results.get(i).node.getId());
         }
     }
 
     public void testNodeNotAvailable() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
         // node2 is not available
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -239,7 +237,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                     }
                     new Thread(() -> {
                         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
                         testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                             new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                         searchActionListener.onResponse(testSearchPhaseResult);
@@ -281,7 +279,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
                 assertNull(results.get(i));
             } else {
                 assertNotNull(results.get(i));
-                assertEquals(context[i].getContextId(), results.get(i).getContextId());
+                assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
                 assertEquals(context[i].getNode(), results.get(i).node.getId());
             }
         }
@@ -289,11 +287,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testShardFailures() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("",17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -313,11 +311,11 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                                                    SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener)
                 {
                     new Thread(() -> {
-                        if (internalRequest.contextId().getId() == 17) {
+                        if (internalRequest.id() == 17) {
                             searchActionListener.onFailure(new IllegalArgumentException("BOOM on shard"));
                         } else {
                             SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                                new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
+                                new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
                             testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                                 new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                             searchActionListener.onResponse(testSearchPhaseResult);
@@ -355,11 +353,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
         ScrollIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
-            if (context[i].getContextId().getId() == 17) {
+            if (context[i].getScrollId() == 17) {
                 assertNull(results.get(i));
             } else {
                 assertNotNull(results.get(i));
-                assertEquals(context[i].getContextId(), results.get(i).getContextId());
+                assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
                 assertEquals(context[i].getNode(), results.get(i).node.getId());
             }
         }
@@ -367,11 +365,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testAllShardsFailed() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
+            new ScrollIdForNode(null, "node1", 1),
+            new ScrollIdForNode(null, "node2", 2),
+            new ScrollIdForNode(null, "node3", 17),
+            new ScrollIdForNode(null, "node1", 0),
+            new ScrollIdForNode(null, "node3", 0));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
index f661cf8e7e8a7..466f56a792491 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -33,7 +32,6 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -59,13 +57,12 @@ public void testSerialization() throws Exception {
 
     public void testInternalScrollSearchRequestSerialization() throws IOException {
         SearchScrollRequest searchScrollRequest = createSearchScrollRequest();
-        InternalScrollSearchRequest internalScrollSearchRequest =
-            new InternalScrollSearchRequest(searchScrollRequest, new SearchContextId(UUIDs.randomBase64UUID(), randomLong()));
+        InternalScrollSearchRequest internalScrollSearchRequest = new InternalScrollSearchRequest(searchScrollRequest, randomLong());
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             internalScrollSearchRequest.writeTo(output);
             try (StreamInput in = output.bytes().streamInput()) {
                 InternalScrollSearchRequest deserializedRequest = new InternalScrollSearchRequest(in);
-                assertEquals(deserializedRequest.contextId().getId(), internalScrollSearchRequest.contextId().getId());
+                assertEquals(deserializedRequest.id(), internalScrollSearchRequest.id());
                 assertEquals(deserializedRequest.scroll(), internalScrollSearchRequest.scroll());
                 assertNotSame(deserializedRequest, internalScrollSearchRequest);
             }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java
deleted file mode 100644
index 8ee1718023e69..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.search;
-
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.EqualsHashCodeTestUtils;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-public class SearchShardTests extends ESTestCase {
-
-   public void testEqualsAndHashcode() {
-       String index = randomAlphaOfLengthBetween(5, 10);
-       SearchShard searchShard = new SearchShard(randomBoolean() ? null : randomAlphaOfLengthBetween(3, 10),
-           new ShardId(index, index + "-uuid", randomIntBetween(0, 1024)));
-       EqualsHashCodeTestUtils.checkEqualsAndHashCode(searchShard,
-           s -> new SearchShard(s.getClusterAlias(), s.getShardId()),
-           s -> {
-            if (randomBoolean()) {
-                return new SearchShard(s.getClusterAlias() == null ? randomAlphaOfLengthBetween(3, 10) : null, s.getShardId());
-            } else {
-                String indexName = s.getShardId().getIndexName();
-                int shardId = s.getShardId().getId();
-                if (randomBoolean()) {
-                    indexName += randomAlphaOfLength(5);
-                } else {
-                    shardId += randomIntBetween(1, 1024);
-                }
-                return new SearchShard(s.getClusterAlias(), new ShardId(indexName, indexName + "-uuid", shardId));
-            }
-           });
-   }
-
-   public void testCompareTo() {
-       List<SearchShard> searchShards = new ArrayList<>();
-       Index index0 = new Index("index0", "index0-uuid");
-       Index index1 = new Index("index1", "index1-uuid");
-       searchShards.add(new SearchShard(null, new ShardId(index0, 0)));
-       searchShards.add(new SearchShard(null, new ShardId(index1, 0)));
-       searchShards.add(new SearchShard(null, new ShardId(index0, 1)));
-       searchShards.add(new SearchShard(null, new ShardId(index1, 1)));
-       searchShards.add(new SearchShard(null, new ShardId(index0, 2)));
-       searchShards.add(new SearchShard(null, new ShardId(index1, 2)));
-       searchShards.add(new SearchShard("", new ShardId(index0, 0)));
-       searchShards.add(new SearchShard("", new ShardId(index1, 0)));
-       searchShards.add(new SearchShard("", new ShardId(index0, 1)));
-       searchShards.add(new SearchShard("", new ShardId(index1, 1)));
-
-       searchShards.add(new SearchShard("remote0", new ShardId(index0, 0)));
-       searchShards.add(new SearchShard("remote0", new ShardId(index1, 0)));
-       searchShards.add(new SearchShard("remote0", new ShardId(index0, 1)));
-       searchShards.add(new SearchShard("remote0", new ShardId(index0, 2)));
-       searchShards.add(new SearchShard("remote1", new ShardId(index0, 0)));
-       searchShards.add(new SearchShard("remote1", new ShardId(index1, 0)));
-       searchShards.add(new SearchShard("remote1", new ShardId(index0, 1)));
-       searchShards.add(new SearchShard("remote1", new ShardId(index1, 1)));
-
-       List<SearchShard> sorted = new ArrayList<>(searchShards);
-       Collections.sort(sorted);
-       assertEquals(searchShards, sorted);
-   }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
index c9d58c7dc9090..a666f45c34311 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -24,13 +24,10 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class TransportSearchHelperTests extends ESTestCase {
 
     public void testParseScrollId() throws IOException {
@@ -38,48 +35,29 @@ public void testParseScrollId() throws IOException {
         DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("x", 1), node1);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("y", 12), node2);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("z", 42), node3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
         array.setOnce(2, testSearchPhaseResult3);
 
-        boolean includeUUID = randomBoolean();
-        String scrollId = TransportSearchHelper.buildScrollId(array, includeUUID);
+        String scrollId = TransportSearchHelper.buildScrollId(array);
         ParsedScrollId parseScrollId = TransportSearchHelper.parseScrollId(scrollId);
         assertEquals(3, parseScrollId.getContext().length);
         assertEquals("node_1", parseScrollId.getContext()[0].getNode());
         assertEquals("cluster_x", parseScrollId.getContext()[0].getClusterAlias());
-        assertEquals(1, parseScrollId.getContext()[0].getContextId().getId());
-        if (includeUUID) {
-            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo("x"));
-        } else {
-            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo(""));
-        }
+        assertEquals(1, parseScrollId.getContext()[0].getScrollId());
 
         assertEquals("node_2", parseScrollId.getContext()[1].getNode());
         assertEquals("cluster_y", parseScrollId.getContext()[1].getClusterAlias());
-        assertEquals(12, parseScrollId.getContext()[1].getContextId().getId());
-        if (includeUUID) {
-            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo("y"));
-        } else {
-            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo(""));
-        }
+        assertEquals(12, parseScrollId.getContext()[1].getScrollId());
 
         assertEquals("node_3", parseScrollId.getContext()[2].getNode());
         assertNull(parseScrollId.getContext()[2].getClusterAlias());
-        assertEquals(42, parseScrollId.getContext()[2].getContextId().getId());
-        if (includeUUID) {
-            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo("z"));
-        } else {
-            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo(""));
-        }
+        assertEquals(42, parseScrollId.getContext()[2].getScrollId());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index afb63edfd15e1..eb2f1c9d845a1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.cluster;
 
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -33,13 +31,10 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.MockLogAppender;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
@@ -77,6 +72,7 @@
 public class NodeConnectionsServiceTests extends ESTestCase {
 
     private ThreadPool threadPool;
+    private MockTransport transport;
     private TransportService transportService;
     private Map<DiscoveryNode, CheckedRunnable<Exception>> nodeConnectionBlocks;
 
@@ -305,116 +301,6 @@ public void testOnlyBlocksOnConnectionsToNewNodes() throws Exception {
         }
     }
 
-    @TestLogging(reason="testing that DEBUG-level logging is reasonable", value="org.elasticsearch.cluster.NodeConnectionsService:DEBUG")
-    public void testDebugLogging() throws IllegalAccessException {
-        final DeterministicTaskQueue deterministicTaskQueue
-            = new DeterministicTaskQueue(builder().put(NODE_NAME_SETTING.getKey(), "node").build(), random());
-
-        MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool());
-        TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool());
-        transportService.start();
-        transportService.acceptIncomingRequests();
-
-        final NodeConnectionsService service
-            = new NodeConnectionsService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), transportService);
-        service.start();
-
-        final List<DiscoveryNode> allNodes = generateNodes();
-        final DiscoveryNodes targetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
-        service.connectToNodes(targetNodes, () -> {});
-        deterministicTaskQueue.runAllRunnableTasks();
-
-        // periodic reconnections to unexpectedly-disconnected nodes are logged
-        final Set<DiscoveryNode> disconnectedNodes = new HashSet<>(randomSubsetOf(allNodes));
-        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
-            transportService.disconnectFromNode(disconnectedNode);
-        }
-        MockLogAppender appender = new MockLogAppender();
-        try {
-            appender.start();
-            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
-            for (DiscoveryNode targetNode : targetNodes) {
-                if (disconnectedNodes.contains(targetNode)) {
-                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connecting to " + targetNode));
-                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connected to " + targetNode));
-                } else {
-                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connecting to " + targetNode));
-                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connected to " + targetNode));
-                }
-            }
-
-            runTasksUntil(deterministicTaskQueue, CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(Settings.EMPTY).millis());
-            appender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
-            appender.stop();
-        }        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
-            transportService.disconnectFromNode(disconnectedNode);
-        }
-
-        // changes to the expected set of nodes are logged, including reconnections to any unexpectedly-disconnected nodes
-        final DiscoveryNodes newTargetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
-        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
-            transportService.disconnectFromNode(disconnectedNode);
-        }
-        appender = new MockLogAppender();
-        try {
-            appender.start();
-            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
-            for (DiscoveryNode targetNode : targetNodes) {
-                if (disconnectedNodes.contains(targetNode) && newTargetNodes.get(targetNode.getId()) != null) {
-                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connecting to " + targetNode));
-                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connected to " + targetNode));
-                } else {
-                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connecting to " + targetNode));
-                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connected to " + targetNode));
-                }
-                if (newTargetNodes.get(targetNode.getId()) == null) {
-                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("disconnected from " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "disconnected from " + targetNode));
-                }
-            }
-            for (DiscoveryNode targetNode : newTargetNodes) {
-                appender.addExpectation(new MockLogAppender.UnseenEventExpectation("disconnected from " + targetNode,
-                    "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                    "disconnected from " + targetNode));
-                if (targetNodes.get(targetNode.getId()) == null) {
-                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connecting to " + targetNode));
-                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
-                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
-                        "connected to " + targetNode));
-                }
-            }
-
-            service.disconnectFromNodesExcept(newTargetNodes);
-            service.connectToNodes(newTargetNodes, () -> {});
-            deterministicTaskQueue.runAllRunnableTasks();
-            appender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
-            appender.stop();
-        }
-    }
-
     private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long endTimeMillis) {
         while (deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) {
             if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) {
@@ -453,8 +339,9 @@ public void setUp() throws Exception {
         super.setUp();
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
         this.threadPool = threadPool;
+        this.transport = new MockTransport(threadPool);
         nodeConnectionBlocks = newConcurrentMap();
-        transportService = new TestTransportService(new MockTransport(threadPool), threadPool);
+        transportService = new TestTransportService(transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
index a800a7adf093b..ef7567ea5df91 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
@@ -68,7 +68,7 @@ public void testJoinDeduplication() {
         // check that sending a join to node1 works
         Optional<Join> optionalJoin1 = randomBoolean() ? Optional.empty() :
             Optional.of(new Join(localNode, node1, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
-        joinHelper.sendJoinRequest(node1, 0L, optionalJoin1);
+        joinHelper.sendJoinRequest(node1, optionalJoin1);
         CapturedRequest[] capturedRequests1 = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests1.length, equalTo(1));
         CapturedRequest capturedRequest1 = capturedRequests1[0];
@@ -79,14 +79,14 @@ public void testJoinDeduplication() {
         // check that sending a join to node2 works
         Optional<Join> optionalJoin2 = randomBoolean() ? Optional.empty() :
             Optional.of(new Join(localNode, node2, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
-        joinHelper.sendJoinRequest(node2, 0L, optionalJoin2);
+        joinHelper.sendJoinRequest(node2, optionalJoin2);
         CapturedRequest[] capturedRequests2 = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests2.length, equalTo(1));
         CapturedRequest capturedRequest2 = capturedRequests2[0];
         assertEquals(node2, capturedRequest2.node);
 
         // check that sending another join to node1 is a noop as the previous join is still in progress
-        joinHelper.sendJoinRequest(node1, 0L, optionalJoin1);
+        joinHelper.sendJoinRequest(node1, optionalJoin1);
         assertThat(capturingTransport.getCapturedRequestsAndClear().length, equalTo(0));
 
         // complete the previous join to node1
@@ -97,7 +97,7 @@ public void testJoinDeduplication() {
         }
 
         // check that sending another join to node1 now works again
-        joinHelper.sendJoinRequest(node1, 0L, optionalJoin1);
+        joinHelper.sendJoinRequest(node1, optionalJoin1);
         CapturedRequest[] capturedRequests1a = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests1a.length, equalTo(1));
         CapturedRequest capturedRequest1a = capturedRequests1a[0];
@@ -106,7 +106,7 @@ public void testJoinDeduplication() {
         // check that sending another join to node2 works if the optionalJoin is different
         Optional<Join> optionalJoin2a = optionalJoin2.isPresent() && randomBoolean() ? Optional.empty() :
             Optional.of(new Join(localNode, node2, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
-        joinHelper.sendJoinRequest(node2, 0L, optionalJoin2a);
+        joinHelper.sendJoinRequest(node2, optionalJoin2a);
         CapturedRequest[] capturedRequests2a = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests2a.length, equalTo(1));
         CapturedRequest capturedRequest2a = capturedRequests2a[0];
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java
index fb99649892ae2..cb89c7c0fea33 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java
@@ -179,18 +179,13 @@ public void testJoinRequestEqualsHashCodeSerialization() {
         Join initialJoin = new Join(createNode(randomAlphaOfLength(10)), createNode(randomAlphaOfLength(10)), randomNonNegativeLong(),
             randomNonNegativeLong(), randomNonNegativeLong());
         JoinRequest initialJoinRequest = new JoinRequest(initialJoin.getSourceNode(),
-            randomNonNegativeLong(), randomBoolean() ? Optional.empty() : Optional.of(initialJoin));
+            randomBoolean() ? Optional.empty() : Optional.of(initialJoin));
         // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(initialJoinRequest,
                 (CopyFunction<JoinRequest>) joinRequest -> copyWriteable(joinRequest, writableRegistry(), JoinRequest::new),
             joinRequest -> {
                 if (randomBoolean() && joinRequest.getOptionalJoin().isPresent() == false) {
-                    return new JoinRequest(createNode(randomAlphaOfLength(10)),
-                        joinRequest.getMinimumTerm(), joinRequest.getOptionalJoin());
-                } else if (randomBoolean()) {
-                    return new JoinRequest(joinRequest.getSourceNode(),
-                        randomValueOtherThan(joinRequest.getMinimumTerm(), ESTestCase::randomNonNegativeLong),
-                        joinRequest.getOptionalJoin());
+                    return new JoinRequest(createNode(randomAlphaOfLength(20)), joinRequest.getOptionalJoin());
                 } else {
                     // change OptionalJoin
                     final Optional<Join> newOptionalJoin;
@@ -200,7 +195,7 @@ public void testJoinRequestEqualsHashCodeSerialization() {
                         newOptionalJoin = Optional.of(new Join(joinRequest.getSourceNode(), createNode(randomAlphaOfLength(10)),
                             randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
                     }
-                    return new JoinRequest(joinRequest.getSourceNode(), joinRequest.getMinimumTerm(), newOptionalJoin);
+                    return new JoinRequest(joinRequest.getSourceNode(), newOptionalJoin);
                 }
             });
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index ac3075597dc57..60575f7487676 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -74,7 +74,6 @@
 import static org.elasticsearch.transport.TransportService.HANDSHAKE_ACTION_NAME;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class NodeJoinTests extends ESTestCase {
 
@@ -279,8 +278,7 @@ public void testJoinWithHigherTermElectsLeader() {
         assertFalse(isLocalNodeElectedMaster());
         assertNull(coordinator.getStateForMasterService().nodes().getMasterNodeId());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture fut = joinNodeAsync(new JoinRequest(node1, newTerm,
-            Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
+        SimpleFuture fut = joinNodeAsync(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
         assertEquals(Coordinator.Mode.LEADER, coordinator.getMode());
         assertNull(coordinator.getStateForMasterService().nodes().getMasterNodeId());
         deterministicTaskQueue.runAllRunnableTasks();
@@ -300,8 +298,7 @@ public void testJoinWithHigherTermButBetterStateGetsRejected() {
         long newTerm = initialTerm + randomLongBetween(1, 10);
         long higherVersion = initialVersion + randomLongBetween(1, 10);
         expectThrows(CoordinationStateRejectedException.class,
-            () -> joinNodeAndRun(new JoinRequest(node1, newTerm,
-                Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion)))));
+            () -> joinNodeAndRun(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion)))));
         assertFalse(isLocalNodeElectedMaster());
     }
 
@@ -315,7 +312,7 @@ public void testJoinWithHigherTermButBetterStateStillElectsMasterThroughSelfJoin
         assertFalse(isLocalNodeElectedMaster());
         long newTerm = initialTerm + randomLongBetween(1, 10);
         long higherVersion = initialVersion + randomLongBetween(1, 10);
-        joinNodeAndRun(new JoinRequest(node1, newTerm, Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion))));
+        joinNodeAndRun(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion))));
         assertTrue(isLocalNodeElectedMaster());
     }
 
@@ -328,32 +325,14 @@ public void testJoinElectedLeader() {
             new VotingConfiguration(Collections.singleton(node0.getId()))));
         assertFalse(isLocalNodeElectedMaster());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        joinNodeAndRun(new JoinRequest(node0, newTerm, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
+        joinNodeAndRun(new JoinRequest(node0, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
         assertFalse(clusterStateHasNode(node1));
-        joinNodeAndRun(new JoinRequest(node1, newTerm, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
+        joinNodeAndRun(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
         assertTrue(clusterStateHasNode(node1));
     }
 
-    public void testJoinElectedLeaderWithHigherTerm() {
-        DiscoveryNode node0 = newNode(0, true);
-        DiscoveryNode node1 = newNode(1, true);
-        long initialTerm = randomLongBetween(1, 10);
-        long initialVersion = randomLongBetween(1, 10);
-        setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
-            new VotingConfiguration(Collections.singleton(node0.getId()))));
-        long newTerm = initialTerm + randomLongBetween(1, 10);
-
-        joinNodeAndRun(new JoinRequest(node0, newTerm, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
-        assertTrue(isLocalNodeElectedMaster());
-
-        long newerTerm = newTerm + randomLongBetween(1, 10);
-        joinNodeAndRun(new JoinRequest(node1, newerTerm, Optional.empty()));
-        assertThat(coordinator.getCurrentTerm(), greaterThanOrEqualTo(newerTerm));
-        assertTrue(isLocalNodeElectedMaster());
-    }
-
     public void testJoinAccumulation() {
         DiscoveryNode node0 = newNode(0, true);
         DiscoveryNode node1 = newNode(1, true);
@@ -364,17 +343,17 @@ public void testJoinAccumulation() {
             new VotingConfiguration(Collections.singleton(node2.getId()))));
         assertFalse(isLocalNodeElectedMaster());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture futNode0 = joinNodeAsync(new JoinRequest(node0, newTerm, Optional.of(
+        SimpleFuture futNode0 = joinNodeAsync(new JoinRequest(node0, Optional.of(
             new Join(node0, node0, newTerm, initialTerm, initialVersion))));
         deterministicTaskQueue.runAllRunnableTasks();
         assertFalse(futNode0.isDone());
         assertFalse(isLocalNodeElectedMaster());
-        SimpleFuture futNode1 = joinNodeAsync(new JoinRequest(node1, newTerm, Optional.of(
+        SimpleFuture futNode1 = joinNodeAsync(new JoinRequest(node1, Optional.of(
             new Join(node1, node0, newTerm, initialTerm, initialVersion))));
         deterministicTaskQueue.runAllRunnableTasks();
         assertFalse(futNode1.isDone());
         assertFalse(isLocalNodeElectedMaster());
-        joinNodeAndRun(new JoinRequest(node2, newTerm, Optional.of(new Join(node2, node0, newTerm, initialTerm, initialVersion))));
+        joinNodeAndRun(new JoinRequest(node2, Optional.of(new Join(node2, node0, newTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
         assertTrue(clusterStateHasNode(node1));
         assertTrue(clusterStateHasNode(node2));
@@ -393,7 +372,7 @@ public void testJoinFollowerWithHigherTerm() throws Exception {
         handleStartJoinFrom(node1, newTerm);
         handleFollowerCheckFrom(node1, newTerm);
         long newerTerm = newTerm + randomLongBetween(1, 10);
-        joinNodeAndRun(new JoinRequest(node1, newerTerm,
+        joinNodeAndRun(new JoinRequest(node1,
             Optional.of(new Join(node1, node0, newerTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
     }
@@ -468,7 +447,7 @@ public void testJoinFollowerFails() throws Exception {
         handleStartJoinFrom(node1, newTerm);
         handleFollowerCheckFrom(node1, newTerm);
         assertThat(expectThrows(CoordinationStateRejectedException.class,
-            () -> joinNodeAndRun(new JoinRequest(node1, newTerm, Optional.empty()))).getMessage(),
+            () -> joinNodeAndRun(new JoinRequest(node1, Optional.empty()))).getMessage(),
             containsString("join target is a follower"));
         assertFalse(isLocalNodeElectedMaster());
     }
@@ -481,8 +460,7 @@ public void testBecomeFollowerFailsPendingJoin() throws Exception {
         setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
             new VotingConfiguration(Collections.singleton(node1.getId()))));
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture fut = joinNodeAsync(new JoinRequest(node0, newTerm,
-            Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
+        SimpleFuture fut = joinNodeAsync(new JoinRequest(node0, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
         deterministicTaskQueue.runAllRunnableTasks();
         assertFalse(fut.isDone());
         assertFalse(isLocalNodeElectedMaster());
@@ -523,7 +501,7 @@ public void testConcurrentJoining() {
         logger.info("Successful voting nodes: {}", successfulNodes);
 
         List<JoinRequest> correctJoinRequests = successfulNodes.stream().map(
-            node -> new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode, newTerm, initialTerm, initialVersion))))
+            node -> new JoinRequest(node, Optional.of(new Join(node, localNode, newTerm, initialTerm, initialVersion))))
             .collect(Collectors.toList());
 
         List<DiscoveryNode> possiblyUnsuccessfulNodes = new ArrayList<>(allNodes);
@@ -534,15 +512,15 @@ public void testConcurrentJoining() {
         List<JoinRequest> possiblyFailingJoinRequests = possiblyUnsuccessfulNodes.stream().map(node -> {
             if (randomBoolean()) {
                 // a correct request
-                return new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode,
+                return new JoinRequest(node, Optional.of(new Join(node, localNode,
                     newTerm, initialTerm, initialVersion)));
             } else if (randomBoolean()) {
                 // term too low
-                return new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode,
+                return new JoinRequest(node, Optional.of(new Join(node, localNode,
                     randomLongBetween(0, initialTerm), initialTerm, initialVersion)));
             } else {
                 // better state
-                return new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode,
+                return new JoinRequest(node, Optional.of(new Join(node, localNode,
                     newTerm, initialTerm, initialVersion + randomLongBetween(1, 10))));
             }
         }).collect(Collectors.toList());
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
index f58753c42c4da..e44805744aa5c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
@@ -105,7 +105,7 @@ public void testHandleNodeJoin_incompatibleClusterState()
         final CompletableFuture<Throwable> future = new CompletableFuture<>();
         DiscoveryNode node = state.nodes().getLocalNode();
 
-        coordinator.sendValidateJoinRequest(stateWithCustomMetaData, new JoinRequest(node, 0L, Optional.empty()),
+        coordinator.sendValidateJoinRequest(stateWithCustomMetaData, new JoinRequest(node, Optional.empty()),
                 new JoinHelper.JoinCallback() {
             @Override
             public void onSuccess() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
deleted file mode 100644
index e8d654603bc12..0000000000000
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.test.AbstractNamedWriteableTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-public class ComponentTemplateMetadataTests extends AbstractNamedWriteableTestCase<ComponentTemplateMetadata> {
-
-    @Override
-    protected ComponentTemplateMetadata createTestInstance() {
-        int count = randomIntBetween(0, 3);
-        Map<String, ComponentTemplate> templateMap = new HashMap<>(count);
-        for (int i = 0; i < count; i++) {
-            templateMap.put(randomAlphaOfLength(4), ComponentTemplateTests.randomInstance());
-        }
-        return new ComponentTemplateMetadata(templateMap);
-    }
-
-    @Override
-    protected ComponentTemplateMetadata mutateInstance(ComponentTemplateMetadata instance) throws IOException {
-        if (instance.componentTemplates().size() == 0) {
-            // Not really much to mutate, so just generate a new one
-            return randomValueOtherThan(instance, this::createTestInstance);
-        }
-        Map<String, ComponentTemplate> templates = new HashMap<>(instance.componentTemplates());
-        Map.Entry<String, ComponentTemplate> newTemplate = instance.componentTemplates().entrySet().iterator().next();
-        if (randomBoolean()) {
-            // Change the key
-            templates.put(randomAlphaOfLength(4), newTemplate.getValue());
-        } else {
-            // Change the value
-            templates.put(newTemplate.getKey(), ComponentTemplateTests.mutateTemplate(newTemplate.getValue()));
-        }
-        return new ComponentTemplateMetadata(templates);
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(Collections.singletonList(new NamedWriteableRegistry.Entry(ComponentTemplateMetadata.class,
-            ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new)));
-    }
-
-    @Override
-    protected Class<ComponentTemplateMetadata> categoryClass() {
-        return ComponentTemplateMetadata.class;
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
deleted file mode 100644
index f0c1b523ac9c1..0000000000000
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.cluster.Diff;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
-public class ComponentTemplateTests extends AbstractDiffableSerializationTestCase<ComponentTemplate> {
-    @Override
-    protected ComponentTemplate makeTestChanges(ComponentTemplate testInstance) {
-        try {
-            return mutateInstance(testInstance);
-        } catch (IOException e) {
-            logger.error(e);
-            fail("mutating should not throw an exception, but got: " + e);
-            return null;
-        }
-    }
-
-    @Override
-    protected Writeable.Reader<Diff<ComponentTemplate>> diffReader() {
-        return ComponentTemplate::readComponentTemplateDiffFrom;
-    }
-
-    @Override
-    protected ComponentTemplate doParseInstance(XContentParser parser) throws IOException {
-        return ComponentTemplate.parse(parser);
-    }
-
-    @Override
-    protected Writeable.Reader<ComponentTemplate> instanceReader() {
-        return ComponentTemplate::new;
-    }
-
-    @Override
-    protected ComponentTemplate createTestInstance() {
-        return randomInstance();
-    }
-
-    public static ComponentTemplate randomInstance() {
-        Settings settings = null;
-        CompressedXContent mappings = null;
-        Map<String, AliasMetaData> aliases = null;
-        if (randomBoolean()) {
-            settings = randomSettings();
-        }
-        if (randomBoolean()) {
-            mappings = randomMappings();
-        }
-        if (randomBoolean()) {
-            aliases = randomAliases();
-        }
-        ComponentTemplate.Template template = new ComponentTemplate.Template(settings, mappings, aliases);
-
-        Map<String, Object> meta = null;
-        if (randomBoolean()) {
-            meta = randomMeta();
-        }
-        return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta);
-    }
-
-    private static Map<String, AliasMetaData> randomAliases() {
-        String aliasName = randomAlphaOfLength(5);
-        AliasMetaData aliasMeta = AliasMetaData.builder(aliasName)
-            .filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
-            .routing(randomBoolean() ? null : randomAlphaOfLength(3))
-            .isHidden(randomBoolean() ? null : randomBoolean())
-            .writeIndex(randomBoolean() ? null : randomBoolean())
-            .build();
-        return Collections.singletonMap(aliasName, aliasMeta);
-    }
-
-    private static CompressedXContent randomMappings() {
-        try {
-            return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}");
-        } catch (IOException e) {
-            fail("got an IO exception creating fake mappings: " + e);
-            return null;
-        }
-    }
-
-    private static Settings randomSettings() {
-        return Settings.builder()
-            .put(randomAlphaOfLength(4), randomAlphaOfLength(10))
-            .build();
-    }
-
-    private static Map<String, Object> randomMeta() {
-        if (randomBoolean()) {
-            return Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4));
-        } else {
-            return Collections.singletonMap(randomAlphaOfLength(5),
-                Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4)));
-        }
-    }
-
-    @Override
-    protected ComponentTemplate mutateInstance(ComponentTemplate orig) throws IOException {
-        return mutateTemplate(orig);
-    }
-
-    public static ComponentTemplate mutateTemplate(ComponentTemplate orig) {
-        switch (randomIntBetween(0, 2)) {
-            case 0:
-                switch (randomIntBetween(0, 2)) {
-                    case 0:
-                        ComponentTemplate.Template ot = orig.template();
-                        return new ComponentTemplate(
-                            new ComponentTemplate.Template(randomValueOtherThan(ot.settings(), ComponentTemplateTests::randomSettings),
-                                ot.mappings(), ot.aliases()),
-                            orig.version(), orig.metadata());
-                    case 1:
-                        ComponentTemplate.Template ot2 = orig.template();
-                        return new ComponentTemplate(
-                            new ComponentTemplate.Template(ot2.settings(),
-                                randomValueOtherThan(ot2.mappings(), ComponentTemplateTests::randomMappings), ot2.aliases()),
-                            orig.version(), orig.metadata());
-                    case 2:
-                        ComponentTemplate.Template ot3 = orig.template();
-                        return new ComponentTemplate(
-                            new ComponentTemplate.Template(ot3.settings(), ot3.mappings(),
-                                randomValueOtherThan(ot3.aliases(), ComponentTemplateTests::randomAliases)),
-                            orig.version(), orig.metadata());
-                    default:
-                        throw new IllegalStateException("illegal randomization branch");
-                }
-            case 1:
-                return new ComponentTemplate(orig.template(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong),
-                    orig.metadata());
-            case 2:
-                return new ComponentTemplate(orig.template(), orig.version(),
-                    randomValueOtherThan(orig.metadata(), ComponentTemplateTests::randomMeta));
-            default:
-                throw new IllegalStateException("illegal randomization branch");
-        }
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 1d0bd3e36c57b..5bd981f246a56 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -724,7 +723,7 @@ public void testFindMappingsWithFilters() throws IOException {
         }
     }
 
-    private static IndexMetaData.Builder buildIndexMetaData(String name, String alias, Boolean writeIndex) {
+    private IndexMetaData.Builder buildIndexMetaData(String name, String alias, Boolean writeIndex) {
         return IndexMetaData.builder(name)
             .settings(settings(Version.CURRENT)).creationDate(randomNonNegativeLong())
             .putAlias(AliasMetaData.builder(alias).writeIndex(writeIndex))
@@ -907,37 +906,4 @@ public void testBuilderRejectsNullInCustoms() {
         final ImmutableOpenMap<String, MetaData.Custom> map = mapBuilder.build();
         assertThat(expectThrows(NullPointerException.class, () -> builder.customs(map)).getMessage(), containsString(key));
     }
-
-    public void testSerialization() throws IOException {
-        final MetaData orig = randomMetaData();
-        final BytesStreamOutput out = new BytesStreamOutput();
-        orig.writeTo(out);
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
-        final MetaData fromStreamMeta = MetaData.readFrom(new NamedWriteableAwareStreamInput(out.bytes().streamInput(),
-            namedWriteableRegistry));
-        assertTrue(MetaData.isGlobalStateEquals(orig, fromStreamMeta));
-    }
-
-    public static MetaData randomMetaData() {
-        return MetaData.builder()
-            .put(buildIndexMetaData("index", "alias", randomBoolean() ? null : randomBoolean()).build(), randomBoolean())
-            .put(IndexTemplateMetaData.builder("template" + randomAlphaOfLength(3))
-                .patterns(Arrays.asList("bar-*", "foo-*"))
-                .settings(Settings.builder()
-                    .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
-                    .build())
-                .build())
-            .persistentSettings(Settings.builder()
-                .put("setting" + randomAlphaOfLength(3), randomAlphaOfLength(4))
-                .build())
-            .transientSettings(Settings.builder()
-                .put("other_setting" + randomAlphaOfLength(3), randomAlphaOfLength(4))
-                .build())
-            .clusterUUID("uuid" + randomAlphaOfLength(3))
-            .clusterUUIDCommitted(randomBoolean())
-            .indexGraveyard(IndexGraveyardTests.createRandom())
-            .version(randomNonNegativeLong())
-            .put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance())
-            .build();
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index b03c41484893f..0338a64b6fe7e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
@@ -117,11 +116,6 @@ public void testSimpleJsonFromAndTo() throws IOException {
                         .putAlias(newAliasMetaDataBuilder("alias-bar1"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
-                .put("component_template", new ComponentTemplate(
-                    new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
-                        new CompressedXContent("{\"baz\":\"eggplant\"}"),
-                        Collections.singletonMap("alias", AliasMetaData.builder("alias").build())),
-                    5L, Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))))
                 .put(IndexMetaData.builder("test12")
                         .settings(settings(Version.CURRENT)
                                 .put("setting1", "value1")
@@ -291,16 +285,6 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").alias(), equalTo("alias-bar3"));
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").indexRouting(), equalTo("routing-bar"));
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").searchRouting(), equalTo("routing-bar"));
-
-        // component template
-        assertNotNull(parsedMetaData.componentTemplates().get("component_template"));
-        assertThat(parsedMetaData.componentTemplates().get("component_template").version(), is(5L));
-        assertThat(parsedMetaData.componentTemplates().get("component_template").metadata(),
-            equalTo(Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))));
-        assertThat(parsedMetaData.componentTemplates().get("component_template").template(),
-            equalTo(new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
-                new CompressedXContent("{\"baz\":\"eggplant\"}"),
-                Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
     }
 
     private static final String MAPPING_SOURCE1 = "{\"mapping1\":{\"text1\":{\"type\":\"string\"}}}";
diff --git a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
index 1105be8a7bf5a..665315e171f5e 100644
--- a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
@@ -19,21 +19,12 @@
 
 package org.elasticsearch.common.util;
 
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.breaker.CircuitBreakingException;
-import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
 public class BitArrayTests extends ESTestCase {
     public void testRandom() {
         try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
@@ -72,31 +63,4 @@ public void testTooBigIsNotSet() {
             }
         }
     }
-
-    public void testClearingDoesntAllocate() {
-        CircuitBreakerService breaker = mock(CircuitBreakerService.class);
-        ByteSizeValue max = new ByteSizeValue(1, ByteSizeUnit.KB);
-        when(breaker.getBreaker(CircuitBreaker.REQUEST)).thenReturn(new NoopCircuitBreaker(CircuitBreaker.REQUEST) {
-            private long total = 0;
-
-            @Override
-            public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
-                total += bytes;
-                if (total > max.getBytes()) {
-                    throw new CircuitBreakingException("test error", bytes, max.getBytes(), Durability.TRANSIENT);
-                }
-                return total;
-            }
-
-            @Override
-            public long addWithoutBreaking(long bytes) {
-                total += bytes;
-                return total;
-            }
-        });
-        BigArrays bigArrays = new BigArrays(null, breaker, CircuitBreaker.REQUEST, true);
-        try (BitArray bitArray = new BitArray(1, bigArrays)) {
-            bitArray.clear(100000000);
-        }
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
index edd74c824af44..ef7831658aa7c 100644
--- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
+++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.Index;
@@ -99,7 +98,7 @@ void syncFlush(String syncId) throws IOException {
             // make sure that we have committed translog; otherwise, we can flush after relaying translog in store recovery
             flush(true, true);
             // make sure that background merges won't happen; otherwise, IndexWriter#hasUncommittedChanges can become true again
-            forceMerge(false, 1, false, false, false, UUIDs.randomBase64UUID());
+            forceMerge(false);
             assertNotNull(indexWriter);
             try (ReleasableLock ignored = writeLock.acquire()) {
                 assertThat(getTranslogStats().getUncommittedOperations(), equalTo(0));
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 976f4d4cd4142..e587c9bc15b81 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -339,7 +339,7 @@ public void testSegmentsWithMergeFlag() throws Exception {
             engine.flush();
             final long gen1 = store.readLastCommittedSegmentsInfo().getGeneration();
             // now, optimize and wait for merges, see that we have no merge flag
-            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(true);
 
             for (Segment segment : engine.segments(false)) {
                 assertThat(segment.getMergeId(), nullValue());
@@ -349,7 +349,7 @@ public void testSegmentsWithMergeFlag() throws Exception {
 
             final boolean flush = randomBoolean();
             final long gen2 = store.readLastCommittedSegmentsInfo().getGeneration();
-            engine.forceMerge(flush, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(flush);
             for (Segment segment : engine.segments(false)) {
                 assertThat(segment.getMergeId(), nullValue());
             }
@@ -1276,7 +1276,7 @@ public void testForceMergeWithSoftDeletesRetention() throws Exception {
             try (Engine.IndexCommitRef safeCommit = engine.acquireSafeIndexCommit()) {
                 safeCommitCheckpoint = Long.parseLong(safeCommit.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
             }
-            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(true, 1, false, false, false);
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             Map<Long, Translog.Operation> ops = readAllOperationsInLucene(engine, mapperService)
                 .stream().collect(Collectors.toMap(Translog.Operation::seqNo, Function.identity()));
@@ -1300,7 +1300,7 @@ public void testForceMergeWithSoftDeletesRetention() throws Exception {
             globalCheckpoint.set(localCheckpoint);
             engine.syncTranslog();
 
-            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(true, 1, false, false, false);
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             assertThat(readAllOperationsInLucene(engine, mapperService), hasSize(liveDocs.size()));
         }
@@ -1364,7 +1364,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
                     safeCommit.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
                 minSeqNoToRetain = Math.min(globalCheckpoint.get() + 1 - retainedExtraOps, safeCommitLocalCheckpoint + 1);
             }
-            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(true, 1, false, false, false);
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             Map<Long, Translog.Operation> ops = readAllOperationsInLucene(engine, mapperService)
                 .stream().collect(Collectors.toMap(Translog.Operation::seqNo, Function.identity()));
@@ -1407,7 +1407,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
                 globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                 engine.syncTranslog();
             }
-            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(true, 1, false, false, false);
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             assertThat(readAllOperationsInLucene(engine, mapperService), hasSize(liveDocsWithSource.size()));
         }
@@ -1443,8 +1443,7 @@ public void run() {
                                 engine.refresh("test");
                                 indexed.countDown();
                                 try {
-                                    engine.forceMerge(
-                                        randomBoolean(), 1, false, randomBoolean(), randomBoolean(), UUIDs.randomBase64UUID());
+                                    engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean());
                                 } catch (IOException e) {
                                     return;
                                 }
@@ -1461,7 +1460,7 @@ public void run() {
                 startGun.countDown();
                 int someIters = randomIntBetween(1, 10);
                 for (int i = 0; i < someIters; i++) {
-                    engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean(), UUIDs.randomBase64UUID());
+                    engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean());
                 }
                 indexed.await();
                 IOUtils.close(engine);
@@ -2839,7 +2838,7 @@ public void run() {
                     try {
                         switch (operation) {
                             case "optimize": {
-                                engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+                                engine.forceMerge(true, 1, false, false, false);
                                 break;
                             }
                             case "refresh": {
@@ -4080,7 +4079,7 @@ public void testRandomOperations() throws Exception {
                 boolean onlyExpungeDeletes = randomBoolean();
                 int maxNumSegments = randomIntBetween(-1, 10);
                 try {
-                    engine.forceMerge(flush, maxNumSegments, onlyExpungeDeletes, false, false, UUIDs.randomBase64UUID());
+                    engine.forceMerge(flush, maxNumSegments, onlyExpungeDeletes, false, false);
                 } catch (IllegalArgumentException e) {
                     assertThat(e.getMessage(), containsString("only_expunge_deletes and max_num_segments are mutually exclusive"));
                     assertThat(onlyExpungeDeletes, is(true));
@@ -4768,7 +4767,7 @@ public void testShouldPeriodicallyFlushAfterMerge() throws Exception {
         engine.index(indexForDoc(doc));
         assertThat(engine.getTranslog().stats().getUncommittedOperations(), equalTo(2));
         engine.refresh("test");
-        engine.forceMerge(false, 1, false, false, false, UUIDs.randomBase64UUID());
+        engine.forceMerge(false, 1, false, false, false);
         assertBusy(() -> {
             // the merge listner runs concurrently after the force merge returned
             assertThat(engine.shouldPeriodicallyFlush(), equalTo(true));
@@ -5014,7 +5013,7 @@ private void assertOperationHistoryInLucene(List<Engine.Operation> operations) t
                     engine.flush();
                 }
                 if (rarely()) {
-                    engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+                    engine.forceMerge(true);
                 }
             }
             MapperService mapperService = createMapperService();
@@ -5093,7 +5092,7 @@ public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException {
                     equalTo(engine.getMinRetainedSeqNo()));
             }
             if (rarely()) {
-                engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
+                engine.forceMerge(randomBoolean());
             }
             try (Closeable ignored = engine.acquireHistoryRetentionLock()) {
                 long minRetainSeqNos = engine.getMinRetainedSeqNo();
@@ -5411,7 +5410,7 @@ public long softUpdateDocuments(Term term, Iterable<? extends Iterable<? extends
                 assertThat(softDeletesEngine.getVersionMap().keySet(), empty());
                 softDeletesEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
                 if (randomBoolean()) {
-                    engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
+                    softDeletesEngine.forceMerge(randomBoolean());
                 }
                 assertThat(getDocIds(softDeletesEngine, true), equalTo(docs));
                 assertConsistentHistoryBetweenTranslogAndLuceneIndex(softDeletesEngine, createMapperService());
@@ -5614,7 +5613,7 @@ public void testPruneAwayDeletedButRetainedIds() throws Exception {
             for (int i = 0; i < numDocs; i++) {
                 index(engine, i);
             }
-            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(true);
             engine.delete(new Engine.Delete("0", newUid("0"), primaryTerm.get()));
             engine.refresh("test");
             // now we have 2 segments since we now added a tombstone plus the old segment with the delete
@@ -5633,7 +5632,7 @@ public void testPruneAwayDeletedButRetainedIds() throws Exception {
             }
 
             // lets force merge the tombstone and the original segment and make sure the doc is still there but the ID term is gone
-            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
+            engine.forceMerge(true);
             engine.refresh("test");
             try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                 IndexReader reader = searcher.getIndexReader();
@@ -5677,7 +5676,7 @@ public void testRecoverFromLocalTranslog() throws Exception {
                         engine.flush();
                     }
                     if (randomInt(100) < 5) {
-                        engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
+                        engine.forceMerge(randomBoolean(), 1, false, false, false);
                     }
                 }
                 if (randomBoolean()) {
diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
index 2583ef577ab36..b1eb9e6ded532 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.store.LockObtainFailedException;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -102,16 +101,10 @@ public void testNoopAfterRegularEngine() throws IOException {
 
     public void testNoOpEngineStats() throws Exception {
         IOUtils.close(engine, store);
-        Settings.Builder settings = Settings.builder()
-            .put(defaultSettings.getSettings())
-            .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0);
-        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
-            IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build());
-        
         final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
         try (Store store = createStore()) {
             Path translogPath = createTempDir();
-            EngineConfig config = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
+            EngineConfig config = config(defaultSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
             final int numDocs = scaledRandomIntBetween(10, 3000);
             int deletions = 0;
             try (InternalEngine engine = createEngine(config)) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index d63a34ab501e3..b276f7fc711af 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
-import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -203,7 +202,7 @@ private void testIgnoreMalfomedForValue(String value, String expectedException)
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(0, fields.length);
-        assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
+        assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
     }
 
     public void testChangeFormat() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 86861901f7a07..8e54a92a22228 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -154,15 +154,11 @@ public void testDotsWithExistingMapper() throws Exception {
             .endObject());
         ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", bytes, XContentType.JSON));
         assertNull(doc.dynamicMappingsUpdate()); // no update!
-
-        IndexableField[] fields = doc.rootDoc().getFields("foo.bar.baz");
-        assertEquals(6, fields.length);
-        assertEquals(123, fields[0].numericValue());
-        assertEquals("123", fields[1].stringValue());
-        assertEquals(456, fields[2].numericValue());
-        assertEquals("456", fields[3].stringValue());
-        assertEquals(789, fields[4].numericValue());
-        assertEquals("789", fields[5].stringValue());
+        String[] values = doc.rootDoc().getValues("foo.bar.baz");
+        assertEquals(3, values.length);
+        assertEquals("123", values[0]);
+        assertEquals("456", values[1]);
+        assertEquals("789", values[2]);
     }
 
     public void testDotsWithExistingNestedMapper() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
index e37523ee730c0..953a24ce00fa8 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.VersionUtils;
 
@@ -53,7 +52,7 @@ private static SortedSet<String> set(String... values) {
     }
 
     void assertFieldNames(Set<String> expected, ParsedDocument doc) {
-        String[] got = TermVectorsService.getValues(doc.rootDoc().getFields("_field_names"));
+        String[] got = doc.rootDoc().getValues("_field_names");
         assertEquals(expected, set(got));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index 37ea96e3df976..a34adaca2f144 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -195,7 +194,7 @@ public void testIgnoreMalformed() throws Exception {
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(0, fields.length);
-        assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
+        assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
     }
 
     public void testNullValue() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index d5f0414708d64..0e25c0a6a212b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
-import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -129,9 +128,6 @@ public void testDefaults() throws Exception {
         fieldType = fields[1].fieldType();
         assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
         assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
-
-        // used by TermVectorsService
-        assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field")));
     }
 
     public void testIgnoreAbove() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
index 7adaf1b2a3233..a1322c6614456 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.randomizedtesting.annotations.Timeout;
+
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.Strings;
@@ -31,7 +32,6 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec;
-import org.elasticsearch.index.termvectors.TermVectorsService;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -250,7 +250,7 @@ public void testIgnoreMalformed() throws Exception {
 
                 IndexableField[] fields = doc.rootDoc().getFields("field");
                 assertEquals(0, fields.length);
-                assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
+                assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
index 0e00bf6677c8f..79a93c04faa1f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
@@ -21,7 +21,14 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.test.VersionUtils;
 import org.mockito.Mockito;
 
 public class TypeFieldTypeTests extends FieldTypeTestCase {
@@ -32,14 +39,40 @@ protected MappedFieldType createDefaultFieldType() {
 
     public void testTermsQuery() throws Exception {
         QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Version indexVersionCreated = VersionUtils.randomIndexCompatibleVersion(random());
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
+        Mockito.when(context.indexVersionCreated()).thenReturn(indexVersionCreated);
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Mockito.when(mapperService.documentMapper()).thenReturn(null);
+        Mockito.when(context.getMapperService()).thenReturn(mapperService);
 
         TypeFieldMapper.TypeFieldType ft = new TypeFieldMapper.TypeFieldType();
         ft.setName(TypeFieldMapper.NAME);
+        Query query = ft.termQuery("my_type", context);
+        assertEquals(new MatchNoDocsQuery(), query);
 
-        Query query = ft.termQuery("_doc", context);
+        DocumentMapper mapper = Mockito.mock(DocumentMapper.class);
+        Mockito.when(mapper.type()).thenReturn("my_type");
+        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
+        query = ft.termQuery("my_type", context);
         assertEquals(new MatchAllDocsQuery(), query);
 
-        query = ft.termQuery("other_type", context);
+        Mockito.when(mapperService.hasNested()).thenReturn(true);
+        query = ft.termQuery("my_type", context);
+        assertEquals(Queries.newNonNestedFilter(), query);
+
+        mapper = Mockito.mock(DocumentMapper.class);
+        Mockito.when(mapper.type()).thenReturn("other_type");
+        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
+        query = ft.termQuery("my_type", context);
         assertEquals(new MatchNoDocsQuery(), query);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
index ed6b58390f645..8e736b12d6a1f 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.IntervalQuery;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
@@ -448,7 +449,7 @@ public void testPrefixes() throws IOException {
         String json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
             "\"prefix\" : { \"prefix\" : \"term\" } } } }";
         IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
-        Query expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.prefix(new BytesRef("term")));
+        Query expected = new IntervalQuery(STRING_FIELD_NAME, XIntervals.prefix(new BytesRef("term")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String no_positions_json = "{ \"intervals\" : { \"" + NO_POSITIONS_FIELD + "\": { " +
@@ -475,7 +476,7 @@ public void testPrefixes() throws IOException {
             "\"prefix\" : { \"prefix\" : \"t\" } } } }";
         builder = (IntervalQueryBuilder) parseQuery(short_prefix_json);
         expected = new IntervalQuery(PREFIXED_FIELD, Intervals.or(
-            Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.wildcard(new BytesRef("t?"))),
+            Intervals.fixField(PREFIXED_FIELD + "._index_prefix", XIntervals.wildcard(new BytesRef("t?"))),
             Intervals.term("t")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
@@ -507,7 +508,7 @@ public void testWildcard() throws IOException {
             "\"wildcard\" : { \"pattern\" : \"Te?m\" } } } }";
 
         IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
-        Query expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.wildcard(new BytesRef("te?m")));
+        Query expected = new IntervalQuery(STRING_FIELD_NAME, XIntervals.wildcard(new BytesRef("te?m")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String no_positions_json = "{ \"intervals\" : { \"" + NO_POSITIONS_FIELD + "\": { " +
@@ -521,14 +522,14 @@ public void testWildcard() throws IOException {
             "\"wildcard\" : { \"pattern\" : \"Te?m\", \"analyzer\" : \"keyword\" } } } }";
 
         builder = (IntervalQueryBuilder) parseQuery(keyword_json);
-        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.wildcard(new BytesRef("Te?m")));
+        expected = new IntervalQuery(STRING_FIELD_NAME, XIntervals.wildcard(new BytesRef("Te?m")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String fixed_field_json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
             "\"wildcard\" : { \"pattern\" : \"Te?m\", \"use_field\" : \"masked_field\" } } } }";
 
         builder = (IntervalQueryBuilder) parseQuery(fixed_field_json);
-        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("te?m"))));
+        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD, XIntervals.wildcard(new BytesRef("te?m"))));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String fixed_field_json_no_positions = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
@@ -543,14 +544,14 @@ public void testWildcard() throws IOException {
 
         builder = (IntervalQueryBuilder) parseQuery(fixed_field_analyzer_json);
         expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD,
-            Intervals.wildcard(new BytesRef("Te?m"))));
+            XIntervals.wildcard(new BytesRef("Te?m"))));
         assertEquals(expected, builder.toQuery(createShardContext()));
     }
 
     private static IntervalsSource buildFuzzySource(String term, String label, int prefixLength, boolean transpositions, int editDistance) {
         FuzzyQuery fq = new FuzzyQuery(new Term("field", term), editDistance, prefixLength, 128, transpositions);
         CompiledAutomaton[] automata = fq.getAutomata();
-        return Intervals.multiterm(automata[automata.length - 1], label);
+        return XIntervals.multiterm(automata[automata.length - 1], label);
     }
 
     public void testFuzzy() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index 852c64fd6fdfd..c9c43a1e90186 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -556,6 +556,13 @@ public void testParseRelation() {
         assertEquals(ShapeRelation.INTERSECTS, builder.relation());
     }
 
+    public void testTypeField() throws IOException {
+        RangeQueryBuilder builder = QueryBuilders.rangeQuery("_type")
+            .from("value1");
+        builder.doToQuery(createShardContext());
+        assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
+    }
+
     /**
      * Range queries should generally be cacheable, at least the ones we create randomly.
      * This test makes sure we also test the non-cacheable cases regularly.
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 3eff0d1f819a5..9482e0bb87d49 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -27,7 +27,6 @@
 
 import java.io.IOException;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -76,9 +75,7 @@ protected void doAssertLuceneQuery(WildcardQueryBuilder queryBuilder, Query quer
 
             assertThat(wildcardQuery.getField(), equalTo(expectedFieldName));
             assertThat(wildcardQuery.getTerm().field(), equalTo(expectedFieldName));
-            // wildcard queries get normalized
-            String text = wildcardQuery.getTerm().text().toLowerCase(Locale.ROOT);
-            assertThat(text, equalTo(text));
+            assertThat(wildcardQuery.getTerm().text(), equalTo(queryBuilder.value()));
         } else {
             Query expected = new MatchNoDocsQuery("unknown field [" + expectedFieldName + "]");
             assertEquals(expected, query);
@@ -141,14 +138,14 @@ public void testTypeField() throws IOException {
         builder.doToQuery(createShardContext());
         assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
     }
-
+    
     public void testRewriteIndexQueryToMatchNone() throws IOException {
         WildcardQueryBuilder query = new WildcardQueryBuilder("_index", "does_not_exist");
         QueryShardContext queryShardContext = createShardContext();
         QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
-    }
-
+    }   
+    
     public void testRewriteIndexQueryNotMatchNone() throws IOException {
         String fullIndexName = getIndex().getName();
         String firstHalfOfIndexName = fullIndexName.substring(0,fullIndexName.length()/2);
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 9401d9dd96d50..0e7bf966c2d0a 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -4104,22 +4104,4 @@ public void testCloseShardWhileEngineIsWarming() throws Exception {
         recoveryThread.join();
         shard.store().close();
     }
-
-    public void testRecordsForceMerges() throws IOException {
-        IndexShard shard = newStartedShard(true);
-        final String initialForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
-        assertThat(initialForceMergeUUID, nullValue());
-        final ForceMergeRequest firstForceMergeRequest = new ForceMergeRequest().maxNumSegments(1);
-        shard.forceMerge(firstForceMergeRequest);
-        final String secondForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
-        assertThat(secondForceMergeUUID, notNullValue());
-        assertThat(secondForceMergeUUID, equalTo(firstForceMergeRequest.forceMergeUUID()));
-        final ForceMergeRequest secondForceMergeRequest = new ForceMergeRequest().maxNumSegments(1);
-        shard.forceMerge(secondForceMergeRequest);
-        final String thirdForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
-        assertThat(thirdForceMergeUUID, notNullValue());
-        assertThat(thirdForceMergeUUID, not(equalTo(secondForceMergeUUID)));
-        assertThat(thirdForceMergeUUID, equalTo(secondForceMergeRequest.forceMergeUUID()));
-        closeShards(shard);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 0a68c54e75b47..560f052664604 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -52,7 +51,6 @@
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ScrollContext;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -120,8 +118,8 @@ public void testPreProcess() throws Exception {
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
-            DefaultSearchContext context1 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 1L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context1 = new DefaultSearchContext(1L, shardSearchRequest, target, searcher, null, indexService,
+                indexShard, bigArrays, null, timeout, null);
             context1.from(300);
 
             // resultWindow greater than maxResultWindow and scrollContext is null
@@ -161,8 +159,8 @@ public void testPreProcess() throws Exception {
                 + "] index level setting."));
 
             // rescore is null but sliceBuilder is not null
-            DefaultSearchContext context2 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 2L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context2 = new DefaultSearchContext(2L, shardSearchRequest, target, searcher,
+                null, indexService, indexShard, bigArrays, null, timeout, null);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
             int numSlices = maxSlicesPerScroll + randomIntBetween(1, 100);
@@ -178,8 +176,8 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.getAliasFilter()).thenReturn(AliasFilter.EMPTY);
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
-            DefaultSearchContext context3 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 3L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context3 = new DefaultSearchContext(3L, shardSearchRequest, target, searcher, null,
+                indexService, indexShard, bigArrays, null, timeout, null);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
             assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query()));
@@ -188,8 +186,8 @@ public void testPreProcess() throws Exception {
             when(queryShardContext.fieldMapper(anyString())).thenReturn(mock(MappedFieldType.class));
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
-            DefaultSearchContext context4 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 4L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context4 = new DefaultSearchContext(4L, shardSearchRequest, target, searcher, null,
+                indexService, indexShard, bigArrays, null, timeout, null);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
             context4.sliceBuilder(new SliceBuilder(0,2)).parsedQuery(parsedQuery).preProcess(false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 7607d519d1d8c..2e61d63c81117 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -78,7 +77,6 @@
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -86,7 +84,6 @@
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
@@ -324,12 +321,12 @@ public void onFailure(Exception e) {
                         SearchPhaseResult searchPhaseResult = result.get();
                         IntArrayList intCursors = new IntArrayList(1);
                         intCursors.add(0);
-                        ShardFetchRequest req = new ShardFetchRequest(searchPhaseResult.getContextId(), intCursors, null/* not a scroll */);
+                        ShardFetchRequest req = new ShardFetchRequest(searchPhaseResult.getRequestId(), intCursors, null/* not a scroll */);
                         PlainActionFuture<FetchSearchResult> listener = new PlainActionFuture<>();
                         service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), listener);
                         listener.get();
                         if (useScroll) {
-                            service.freeContext(searchPhaseResult.getContextId());
+                            service.freeContext(searchPhaseResult.getRequestId());
                         }
                     } catch (ExecutionException ex) {
                         assertThat(ex.getCause(), instanceOf(RuntimeException.class));
@@ -539,47 +536,6 @@ public void testMaxOpenScrollContexts() throws RuntimeException, IOException {
             ex.getMessage());
     }
 
-    public void testOpenScrollContextsConcurrently() throws Exception {
-        createIndex("index");
-        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        final IndexShard indexShard = indicesService.indexServiceSafe(resolveIndex("index")).getShard(0);
-
-        final int maxScrollContexts = SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY);
-        final SearchService searchService = getInstanceFromNode(SearchService.class);
-        Thread[] threads = new Thread[randomIntBetween(2, 8)];
-        CountDownLatch latch = new CountDownLatch(threads.length);
-        for (int i = 0; i < threads.length; i++) {
-            threads[i] = new Thread(() -> {
-                latch.countDown();
-                try {
-                    latch.await();
-                    for (; ; ) {
-                        SearchService.SearchRewriteContext rewriteContext =
-                            searchService.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
-                        try {
-                            searchService.createAndPutContext(rewriteContext);
-                        } catch (ElasticsearchException e) {
-                            assertThat(e.getMessage(), equalTo(
-                                "Trying to create too many scroll contexts. Must be less than or equal to: " +
-                                    "[" + maxScrollContexts + "]. " +
-                                    "This limit can be set by changing the [search.max_open_scroll_context] setting."));
-                            return;
-                        }
-                    }
-                } catch (Exception e) {
-                    throw new AssertionError(e);
-                }
-            });
-            threads[i].setName("elasticsearch[node_s_0][search]");
-            threads[i].start();
-        }
-        for (Thread thread : threads) {
-            thread.join();
-        }
-        assertThat(searchService.getActiveContexts(), equalTo(maxScrollContexts));
-        searchService.freeAllScrollContexts();
-    }
-
     public static class FailOnRewriteQueryPlugin extends Plugin implements SearchPlugin {
         @Override
         public List<QuerySpec<?>> getQueries() {
@@ -973,40 +929,4 @@ public void testDeleteIndexWhileSearch() throws Exception {
             searcher.join();
         }
     }
-
-    public void testLookUpSearchContext() throws Exception {
-        createIndex("index");
-        SearchService searchService = getInstanceFromNode(SearchService.class);
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
-        IndexShard indexShard = indexService.getShard(0);
-        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(
-            OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
-            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-        List<SearchContextId> contextIds = new ArrayList<>();
-        int numContexts = randomIntBetween(1, 10);
-        for (int i = 0; i < numContexts; i++) {
-            SearchService.SearchRewriteContext rewriteContext = searchService.acquireSearcherAndRewrite(shardSearchRequest, indexShard);
-            final SearchContext searchContext = searchService.createContext(rewriteContext);
-            assertThat(searchContext.id().getId(), equalTo((long) (i + 1)));
-            searchService.putContext(searchContext);
-            contextIds.add(searchContext.id());
-        }
-        assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-        while (contextIds.isEmpty() == false) {
-            final SearchContextId contextId = randomFrom(contextIds);
-            assertFalse(searchService.freeContext(new SearchContextId(UUIDs.randomBase64UUID(), contextId.getId())));
-            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-            if (randomBoolean()) {
-                assertTrue(searchService.freeContext(contextId));
-            } else {
-                assertTrue(searchService.freeContext((new SearchContextId("", contextId.getId()))));
-            }
-            contextIds.remove(contextId);
-            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-            assertFalse(searchService.freeContext(new SearchContextId("", contextId.getId())));
-            assertFalse(searchService.freeContext(contextId));
-            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-        }
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
new file mode 100644
index 0000000000000..ddd8c1920b99e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.bucket.missing.Missing;
+import org.elasticsearch.search.aggregations.metrics.Avg;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.missing;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.core.IsNull.notNullValue;
+
+@ESIntegTestCase.SuiteScopeTestCase
+public class MissingIT extends ESIntegTestCase {
+
+    static int numDocs, numDocsMissing, numDocsUnmapped;
+
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("idx")
+                .setMapping("tag", "type=keyword").get());
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        numDocs = randomIntBetween(5, 20);
+        numDocsMissing = randomIntBetween(1, numDocs - 1);
+        for (int i = 0; i < numDocsMissing; i++) {
+            builders.add(client().prepareIndex("idx").setId(""+i).setSource(jsonBuilder()
+                    .startObject()
+                    .field("value", i)
+                    .endObject()));
+        }
+        for (int i = numDocsMissing; i < numDocs; i++) {
+            builders.add(client().prepareIndex("idx").setId(""+i).setSource(jsonBuilder()
+                    .startObject()
+                    .field("tag", "tag1")
+                    .endObject()));
+        }
+
+        createIndex("unmapped_idx");
+        numDocsUnmapped = randomIntBetween(2, 5);
+        for (int i = 0; i < numDocsUnmapped; i++) {
+            builders.add(client().prepareIndex("unmapped_idx").setId(""+i).setSource(jsonBuilder()
+                    .startObject()
+                    .field("value", i)
+                    .endObject()));
+        }
+
+        prepareCreate("empty_bucket_idx").setMapping("value", "type=integer").get();
+        for (int i = 0; i < 2; i++) {
+            builders.add(client().prepareIndex("empty_bucket_idx").setId(""+i).setSource(jsonBuilder()
+                    .startObject()
+                    .field("value", i*2)
+                    .endObject()));
+        }
+
+        indexRandom(true, builders);
+        ensureSearchable();
+    }
+
+    public void testUnmapped() throws Exception {
+        SearchResponse response = client().prepareSearch("unmapped_idx")
+                .addAggregation(missing("missing_tag").field("tag"))
+                .get();
+
+        assertSearchResponse(response);
+
+
+        Missing missing = response.getAggregations().get("missing_tag");
+        assertThat(missing, notNullValue());
+        assertThat(missing.getName(), equalTo("missing_tag"));
+        assertThat(missing.getDocCount(), equalTo((long) numDocsUnmapped));
+    }
+
+    public void testPartiallyUnmapped() throws Exception {
+        SearchResponse response = client().prepareSearch("idx", "unmapped_idx")
+                .addAggregation(missing("missing_tag").field("tag"))
+                .get();
+
+        assertSearchResponse(response);
+
+
+        Missing missing = response.getAggregations().get("missing_tag");
+        assertThat(missing, notNullValue());
+        assertThat(missing.getName(), equalTo("missing_tag"));
+        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped));
+    }
+
+    public void testSimple() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(missing("missing_tag").field("tag"))
+                .get();
+
+        assertSearchResponse(response);
+
+
+        Missing missing = response.getAggregations().get("missing_tag");
+        assertThat(missing, notNullValue());
+        assertThat(missing.getName(), equalTo("missing_tag"));
+        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing));
+    }
+
+    public void testWithSubAggregation() throws Exception {
+        SearchResponse response = client().prepareSearch("idx", "unmapped_idx")
+                .addAggregation(missing("missing_tag").field("tag")
+                        .subAggregation(avg("avg_value").field("value")))
+                .get();
+
+        assertSearchResponse(response);
+
+        assertThat("Not all shards are initialized", response.getSuccessfulShards(), equalTo(response.getTotalShards()));
+
+        Missing missing = response.getAggregations().get("missing_tag");
+        assertThat(missing, notNullValue());
+        assertThat(missing.getName(), equalTo("missing_tag"));
+        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped));
+        assertThat((long) ((InternalAggregation)missing).getProperty("_count"), equalTo((long) numDocsMissing + numDocsUnmapped));
+        assertThat(missing.getAggregations().asList().isEmpty(), is(false));
+
+        long sum = 0;
+        for (int i = 0; i < numDocsMissing; ++i) {
+            sum += i;
+        }
+        for (int i = 0; i < numDocsUnmapped; ++i) {
+            sum += i;
+        }
+        Avg avgValue = missing.getAggregations().get("avg_value");
+        assertThat(avgValue, notNullValue());
+        assertThat(avgValue.getName(), equalTo("avg_value"));
+        assertThat(avgValue.getValue(), equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
+        assertThat((double) ((InternalAggregation)missing).getProperty("avg_value.value"),
+                equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
+    }
+
+    public void testEmptyAggregation() throws Exception {
+        SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
+                .setQuery(matchAllQuery())
+                .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0)
+                        .subAggregation(missing("missing").field("value")))
+                .get();
+
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
+        Histogram histo = searchResponse.getAggregations().get("histo");
+        assertThat(histo, Matchers.notNullValue());
+        Histogram.Bucket bucket = histo.getBuckets().get(1);
+        assertThat(bucket, Matchers.notNullValue());
+
+        Missing missing = bucket.getAggregations().get("missing");
+        assertThat(missing, Matchers.notNullValue());
+        assertThat(missing.getName(), equalTo("missing"));
+        assertThat(missing.getDocCount(), is(0L));
+    }
+
+
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
index dbea1d7f2729e..7685cd9067ddc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
@@ -2077,52 +2077,40 @@ private static long asLong(String dateTime) {
 
     private static Sort buildIndexSort(List<CompositeValuesSourceBuilder<?>> sources, Map<String, MappedFieldType> fieldTypes) {
         List<SortField> sortFields = new ArrayList<>();
-        Map<String, MappedFieldType> remainingFieldTypes = new HashMap<>(fieldTypes);
         for (CompositeValuesSourceBuilder<?> source : sources) {
-            MappedFieldType type = fieldTypes.remove(source.field());
-            remainingFieldTypes.remove(source.field());
-            SortField sortField = sortFieldFrom(type);
-            if (sortField == null) {
-                break;
-            }
-            sortFields.add(sortField);
-            if (sortField instanceof SortedNumericSortField && ((SortedNumericSortField) sortField).getType() == SortField.Type.LONG) {
-                break;
-            }
-        }
-        while (remainingFieldTypes.size() > 0 && randomBoolean()) {
-            // Add extra unused sorts
-            List<String> fields = new ArrayList<>(remainingFieldTypes.keySet());
-            Collections.sort(fields);
-            String field = fields.get(between(0, fields.size() - 1));
-            SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field));
-            if (sortField != null) {
-                sortFields.add(sortField);
-            }
-        }
-        return sortFields.size() > 0 ? new Sort(sortFields.toArray(SortField[]::new)) : null;
-    }
-    
-    private static SortField sortFieldFrom(MappedFieldType type) {
-        if (type instanceof KeywordFieldMapper.KeywordFieldType) {
-            return new SortedSetSortField(type.name(), false);
-        } else if (type instanceof DateFieldMapper.DateFieldType) {
-            return new SortedNumericSortField(type.name(), SortField.Type.LONG, false);
-        } else if (type instanceof NumberFieldMapper.NumberFieldType) {
-            switch (type.typeName()) {
-                case "byte":
-                case "short":
-                case "integer":
-                    return new SortedNumericSortField(type.name(), SortField.Type.INT, false);
-                case "long":
-                    return new SortedNumericSortField(type.name(), SortField.Type.LONG, false);
-                case "float":
-                case "double":
-                    return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false);
-                default:
-                    return null;
+            MappedFieldType type = fieldTypes.get(source.field());
+            if (type instanceof KeywordFieldMapper.KeywordFieldType) {
+                sortFields.add(new SortedSetSortField(type.name(), false));
+            } else if (type instanceof DateFieldMapper.DateFieldType) {
+                sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.LONG, false));
+            } else if (type instanceof NumberFieldMapper.NumberFieldType) {
+                boolean comp = false;
+                switch (type.typeName()) {
+                    case "byte":
+                    case "short":
+                    case "integer":
+                        comp = true;
+                        sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.INT, false));
+                        break;
+
+                    case "long":
+                        sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.LONG, false));
+                        break;
+
+                    case "float":
+                    case "double":
+                        comp = true;
+                        sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false));
+                        break;
+
+                    default:
+                        break;
+                }
+                if (comp == false) {
+                    break;
+                }
             }
         }
-        return null;
+        return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java
index 7657754b85e9e..2059f465c3756 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java
@@ -33,12 +33,9 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
-import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -73,16 +70,6 @@ public abstract class GeoGridAggregatorTestCase<T extends InternalGeoGridBucket>
      */
     protected abstract GeoGridAggregationBuilder createBuilder(String name);
 
-    @Override
-    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
-        return createBuilder("foo").field(fieldName);
-    }
-
-    @Override
-    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
-        return List.of(CoreValuesSourceType.GEOPOINT);
-    }
-
     public void testNoDocs() throws IOException {
         testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), null, geoGrid -> {
             assertEquals(0, geoGrid.getBuckets().size());
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
index 3665fcded7280..dbc3ac3b490c2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
@@ -20,617 +20,174 @@
 package org.elasticsearch.search.aggregations.bucket.missing;
 
 import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.CheckedConsumer;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.mapper.RangeType;
-import org.elasticsearch.script.MockScriptEngine;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
-import org.elasticsearch.search.aggregations.metrics.InternalSum;
-import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
-import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
-import java.util.function.Function;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singleton;
-import static java.util.Collections.singletonMap;
-import static java.util.stream.Collectors.toList;
-import static org.elasticsearch.common.lucene.search.Queries.newMatchAllQuery;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
-import static org.hamcrest.collection.IsEmptyCollection.empty;
 
 public class MissingAggregatorTests extends AggregatorTestCase {
-
-    private static final String VALUE_SCRIPT_PARAMS = "value_script_params";
-    private static final String VALUE_SCRIPT = "value_script";
-    private static final String FIELD_SCRIPT_PARAMS = "field_script_params";
-    private static final String FIELD_SCRIPT = "field_script";
-    private static final long DEFAULT_INC_PARAM = 1;
-    private static final long DEFAULT_THRESHOLD_PARAM = 50;
-
     public void testMatchNoDocs() throws IOException {
-        final int numDocs = randomIntBetween(10, 200);
-
-        final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        fieldType.setName("field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(fieldType.name());
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> {
-                for (int i = 0; i < numDocs; i++) {
-                    writer.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong())));
-                }
-            },
-            (InternalMissing missing) -> {
-                assertEquals(0, missing.getDocCount());
-                assertFalse(AggregationInspectionHelper.hasValue(missing));
-            },
-            singleton(fieldType)
-        );
+        int numDocs = randomIntBetween(10, 200);
+        testBothCases(numDocs,
+            "field",
+            Queries.newMatchAllQuery(),
+            doc -> doc.add(new SortedNumericDocValuesField("field", randomLong())),
+            internalMissing -> {
+                assertEquals(internalMissing.getDocCount(), 0);
+                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
+            });
     }
 
     public void testMatchAllDocs() throws IOException {
         int numDocs = randomIntBetween(10, 200);
-
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        anotherFieldType.setName("another_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(aggFieldType.name());
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> {
-                for (int i = 0; i < numDocs; i++) {
-                    writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
-                }
-            },
-            (InternalMissing missing) -> {
-                assertEquals(numDocs, missing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(missing));
-            },
-            List.of(aggFieldType, anotherFieldType)
-        );
-    }
-
-    public void testMatchSparse() throws IOException {
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        anotherFieldType.setName("another_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(aggFieldType.name());
-
-        final int numDocs = randomIntBetween(100, 200);
-        int docsMissingAggField = 0;
-        final List<Set<IndexableField>> docs = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            if (randomBoolean()) {
-                docs.add(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
-            } else {
-                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
-                docsMissingAggField++;
-            }
-        }
-        final int finalDocsMissingAggField = docsMissingAggField;
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> writer.addDocuments(docs),
-            (InternalMissing missing) -> {
-                assertEquals(finalDocsMissingAggField, missing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(missing));
-            },
-            List.of(aggFieldType, anotherFieldType)
-        );
-    }
-
-    public void testMatchSparseRangeField() throws IOException {
-        final RangeType rangeType = RangeType.DOUBLE;
-        MappedFieldType aggFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType();
-        aggFieldType.setName("agg_field");
-        final MappedFieldType anotherFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType();
-        anotherFieldType.setName("another_field");
-
-        final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true);
-        final BytesRef encodedRange = rangeType.encodeRanges(singleton(range));
-        final BinaryDocValuesField encodedRangeField = new BinaryDocValuesField(aggFieldType.name(), encodedRange);
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(aggFieldType.name());
-
-        final int numDocs = randomIntBetween(100, 200);
-        int docsMissingAggField = 0;
-        final List<Set<IndexableField>> docs = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            if (randomBoolean()) {
-                docs.add(singleton(encodedRangeField));
-            } else {
-                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
-                docsMissingAggField++;
-            }
-        }
-        final int finalDocsMissingAggField = docsMissingAggField;
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> writer.addDocuments(docs),
-            (InternalMissing missing) -> {
-                assertEquals(finalDocsMissingAggField, missing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(missing));
-            },
-            List.of(aggFieldType, anotherFieldType)
-        );
-    }
-
-    public void testSubAggregation() throws IOException {
-        final MappedFieldType missingAggFieldType = new KeywordFieldMapper.Builder("_name").fieldType();
-        missingAggFieldType.setName("missing_agg_field");
-        missingAggFieldType.setHasDocValues(true);
-        final MappedFieldType sumAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        sumAggFieldType.setName("sum_agg_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("missing", null)
-            .field(missingAggFieldType.name())
-            .subAggregation(
-              new SumAggregationBuilder("sum")
-                .field(sumAggFieldType.name())
-            );
-
-        final int numDocs = randomIntBetween(10, 20);
-        int docsMissingAggField = 0;
-        long expectedSum = 0;
-        final List<Set<IndexableField>> docs = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            final Set<IndexableField> doc = new HashSet<>();
-            final long sumFieldValue = randomLongBetween(0, 1000);
-            doc.add(new SortedNumericDocValuesField(sumAggFieldType.name(), sumFieldValue));
-            if (randomBoolean()) {
-                doc.add(new SortedDocValuesField(missingAggFieldType.name(), new BytesRef(randomUnicodeOfLengthBetween(2, 20))));
-            } else {
-                docsMissingAggField++;
-                expectedSum += sumFieldValue;
-            }
-            docs.add(doc);
-        }
-        final int finalDocsMissingAggField = docsMissingAggField;
-        final long finalExpectedSum = expectedSum;
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> writer.addDocuments(docs),
-            (InternalMissing internalMissing) -> {
-                assertEquals(finalDocsMissingAggField, internalMissing.getDocCount());
+        testBothCases(numDocs,
+            "field",
+            Queries.newMatchAllQuery(),
+            doc -> doc.add(new SortedNumericDocValuesField("another_field", randomLong())),
+            internalMissing -> {
+                assertEquals(internalMissing.getDocCount(), numDocs);
                 assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
-
-                assertThat(internalMissing.getAggregations().asList(), not(empty()));
-                final InternalSum internalSum = internalMissing.getAggregations().get("sum");
-                assertEquals(finalExpectedSum, internalSum.getValue(), 0d);
-                assertTrue(AggregationInspectionHelper.hasValue(internalSum));
-            },
-            List.of(missingAggFieldType, sumAggFieldType)
-        );
+            });
     }
 
-    public void testEmptyBucket() throws IOException {
-        final MappedFieldType histoAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        histoAggFieldType.setName("histo_agg_field");
-        final MappedFieldType missingAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        missingAggFieldType.setName("missing_agg_field");
-
-        final HistogramAggregationBuilder builder = new HistogramAggregationBuilder("histo")
-            .field(histoAggFieldType.name())
-            .interval(1)
-            .minDocCount(0)
-            .subAggregation(
-              new MissingAggregationBuilder("missing", null)
-                .field(missingAggFieldType.name())
-            );
-
-        testCaseWithReduce(
-            newMatchAllQuery(),
-            builder,
-            writer -> {
-                writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 0)));
-                writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 2)));
-            },
-            (InternalHistogram histogram) -> {
-                assertThat(histogram.getBuckets(), hasSize(3));
-
-                {
-                    assertThat(histogram.getBuckets().get(0), notNullValue());
-                    final InternalMissing missing = histogram.getBuckets().get(0).getAggregations().get("missing");
-                    assertEquals(1, missing.getDocCount());
-                    assertTrue(AggregationInspectionHelper.hasValue(missing));
-                }
-                {
-                    assertThat(histogram.getBuckets().get(1), notNullValue());
-                    final InternalMissing missing = histogram.getBuckets().get(1).getAggregations().get("missing");
-                    assertEquals(0, missing.getDocCount());
-                    assertFalse(AggregationInspectionHelper.hasValue(missing));
-                }
-                {
-                    assertThat(histogram.getBuckets().get(2), notNullValue());
-                    final InternalMissing missing = histogram.getBuckets().get(2).getAggregations().get("missing");
-                    assertEquals(1, missing.getDocCount());
-                    assertTrue(AggregationInspectionHelper.hasValue(missing));
-                }
-            },
-            List.of(histoAggFieldType, missingAggFieldType),
-            true
-        );
-    }
-
-    public void testUnmappedWithoutMissingParam() throws IOException {
-        final int numDocs = randomIntBetween(10, 20);
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field("unknown_field");
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> {
-                for (int i = 0; i < numDocs; i++) {
-                    writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
-                }
-            },
-            (InternalMissing missing) -> {
-                assertEquals(numDocs, missing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(missing));
-            },
-            singleton(aggFieldType)
-        );
-    }
-
-    public void testUnmappedWithMissingParam() throws IOException {
-        final int numDocs = randomIntBetween(10, 20);
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field("unknown_field")
-            .missing(randomLong());
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> {
-                for (int i = 0; i < numDocs; i++) {
-                    writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
+    public void testMatchSparse() throws IOException {
+        int numDocs = randomIntBetween(100, 200);
+        final AtomicInteger count = new AtomicInteger();
+        testBothCases(numDocs,
+            "field",
+            Queries.newMatchAllQuery(),
+            doc -> {
+                if (randomBoolean()) {
+                    doc.add(new SortedNumericDocValuesField("another_field", randomLong()));
+                    count.incrementAndGet();
+                } else {
+                    doc.add(new SortedNumericDocValuesField("field", randomLong()));
                 }
             },
-            (InternalMissing missing) -> {
-                assertEquals(0, missing.getDocCount());
-                assertFalse(AggregationInspectionHelper.hasValue(missing));
-            },
-            singleton(aggFieldType)
-        );
-    }
-
-    public void testSingleValuedFieldPartiallyUnmapped() throws IOException {
-        final int numDocs = randomIntBetween(10, 20);
-
-        final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        fieldType.setName("field");
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(fieldType.name());
-
-        try (Directory mappedDirectory = newDirectory(); Directory unmappedDirectory = newDirectory()) {
-            try (RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory)) {
-                for (int i = 0; i < numDocs; i++) {
-                    mappedWriter.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong())));
-                }
-            }
-
-            new RandomIndexWriter(random(), unmappedDirectory).close();
-
-            try (IndexReader mappedReader = DirectoryReader.open(mappedDirectory);
-                 IndexReader unmappedReader = DirectoryReader.open(unmappedDirectory);
-                 MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) {
-
-                final IndexSearcher searcher = newSearcher(multiReader, true, true);
-
-                final InternalMissing internalMissing = searchAndReduce(searcher, newMatchAllQuery(), builder, fieldType);
-                assertEquals(0, internalMissing.getDocCount());
-                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
-            }
-        }
+            internalMissing -> {
+                assertEquals(internalMissing.getDocCount(), count.get());
+                count.set(0);
+                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            });
     }
 
-    public void testMissingParam() throws IOException {
-        final int numDocs = randomIntBetween(10, 20);
-
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        anotherFieldType.setName("another_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(aggFieldType.name())
-            .missing(randomLong());
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> {
-                for (int i = 0; i < numDocs; i++) {
-                    writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
+    public void testMatchSparseRangeField() throws IOException {
+        int numDocs = randomIntBetween(100, 200);
+        final AtomicInteger count = new AtomicInteger();
+        final String fieldName = "field";
+        RangeType rangeType = RangeType.DOUBLE;
+        final BinaryDocValuesField field = new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(
+            new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true))));
+        MappedFieldType fieldType = new RangeFieldMapper.Builder(fieldName, rangeType).fieldType();
+        fieldType.setName(fieldName);
+        testBothCases(numDocs,
+            fieldName,
+            Queries.newMatchAllQuery(),
+            doc -> {
+                if (randomBoolean()) {
+                    doc.add(new SortedNumericDocValuesField("another_field", randomLong()));
+                    count.incrementAndGet();
+                } else {
+                    doc.add(field);
                 }
             },
-            (InternalMissing missing) -> {
-                assertEquals(0, missing.getDocCount());
-                assertFalse(AggregationInspectionHelper.hasValue(missing));
-            },
-            List.of(aggFieldType, anotherFieldType)
-        );
-    }
-
-    public void testMultiValuedField() throws IOException {
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        anotherFieldType.setName("another_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(aggFieldType.name());
-
-        final int numDocs = randomIntBetween(100, 200);
-        int docsMissingAggField = 0;
-        final List<Set<IndexableField>> docs = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            if (randomBoolean()) {
-                final long randomLong = randomLong();
-                docs.add(Set.of(
-                    new SortedNumericDocValuesField(aggFieldType.name(), randomLong),
-                    new SortedNumericDocValuesField(aggFieldType.name(), randomLong + 1)
-                ));
-            } else {
-                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
-                docsMissingAggField++;
-            }
-        }
-        final int finalDocsMissingAggField = docsMissingAggField;
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> writer.addDocuments(docs),
-            (InternalMissing missing) -> {
-                assertEquals(finalDocsMissingAggField, missing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(missing));
-            },
-            List.of(aggFieldType, anotherFieldType)
-        );
-    }
-
-    public void testSingleValuedFieldWithValueScript() throws IOException {
-        valueScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, emptyMap()));
-    }
-
-    public void testSingleValuedFieldWithValueScriptWithParams() throws IOException {
-        valueScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_PARAMS, singletonMap("inc", 10)));
-    }
-
-    private void valueScriptTestCase(Script script) throws IOException {
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        anotherFieldType.setName("another_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .field(aggFieldType.name())
-            .script(script);
-
-        final int numDocs = randomIntBetween(100, 200);
-        int docsMissingAggField = 0;
-        final List<Set<IndexableField>> docs = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            if (randomBoolean()) {
-                docs.add(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
-            } else {
-                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
-                docsMissingAggField++;
-            }
-        }
-        final int finalDocsMissingField = docsMissingAggField;
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> writer.addDocuments(docs),
-            (InternalMissing missing) -> {
-                assertEquals(finalDocsMissingField, missing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(missing));
-            },
-            List.of(aggFieldType, anotherFieldType)
-        );
-    }
-
-    public void testMultiValuedFieldWithFieldScriptWithParams() throws IOException {
-        final long threshold = 10;
-        final Map<String, Object> params = Map.of("field", "agg_field", "threshold", threshold);
-        fieldScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_PARAMS, params), threshold);
-    }
-
-    public void testMultiValuedFieldWithFieldScript() throws IOException {
-        fieldScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT, singletonMap("field", "agg_field")),
-            DEFAULT_THRESHOLD_PARAM);
+            internalMissing -> {
+                assertEquals(internalMissing.getDocCount(), count.get());
+                count.set(0);
+                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            }, fieldType);
     }
 
-    private void fieldScriptTestCase(Script script, long threshold) throws IOException {
-        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
-        aggFieldType.setName("agg_field");
-
-        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
-            .script(script);
 
-        final int numDocs = randomIntBetween(100, 200);
-        int docsBelowThreshold = 0;
-        final List<Set<IndexableField>> docs = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            final long firstValue = randomLongBetween(0, 100);
-            final long secondValue = firstValue + 1;
-            if (firstValue < threshold && secondValue < threshold) {
-                docsBelowThreshold++;
-            }
-            docs.add(Set.of(
-                new SortedNumericDocValuesField(aggFieldType.name(), firstValue),
-                new SortedNumericDocValuesField(aggFieldType.name(), secondValue)
-            ));
-        }
-        final int finalDocsBelowThreshold = docsBelowThreshold;
-
-        testCase(
-            newMatchAllQuery(),
-            builder,
-            writer -> writer.addDocuments(docs),
-            (InternalMissing missing) -> {
-                assertEquals(finalDocsBelowThreshold, missing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(missing));
+    public void testMissingField() throws IOException {
+        int numDocs = randomIntBetween(10, 20);
+        testBothCases(numDocs,
+            "unknown_field",
+            Queries.newMatchAllQuery(),
+            doc -> {
+                doc.add(new SortedNumericDocValuesField("field", randomLong()));
             },
-            singleton(aggFieldType)
-        );
-    }
-
-    private <B extends AggregationBuilder, I extends InternalAggregation> void testCase(
-            Query query,
-            B builder,
-            CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
-            Consumer<I> verify,
-            Collection<MappedFieldType> fieldTypes) throws IOException {
-
-        testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false);
-        testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true);
-    }
-
-    private <B extends AggregationBuilder, I extends InternalAggregation> void testCaseWithReduce(
-            Query query,
-            B builder,
-            CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
-            Consumer<I> verify,
-            Collection<MappedFieldType> fieldTypes,
-            boolean reduced) throws IOException {
-
+            internalMissing -> {
+                assertEquals(internalMissing.getDocCount(), numDocs);
+                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            });
+    }
+
+    private void testBothCases(int numDocs,
+                               String fieldName,
+                               Query query,
+                               Consumer<Document> consumer,
+                               Consumer<InternalMissing> verify) throws IOException {
+        NumberFieldMapper.Builder mapperBuilder = new NumberFieldMapper.Builder("_name",
+            NumberFieldMapper.NumberType.LONG);
+        final MappedFieldType fieldType = mapperBuilder.fieldType();
+        fieldType.setHasDocValues(true);
+        fieldType.setName(fieldName);
+        testBothCases(numDocs, fieldName, query, consumer, verify, fieldType);
+    }
+
+    private void testBothCases(int numDocs,
+                               String fieldName,
+                               Query query,
+                               Consumer<Document> consumer,
+                               Consumer<InternalMissing> verify,
+                               MappedFieldType fieldType) throws IOException {
+        executeTestCase(numDocs, fieldName, query, consumer, verify, false, fieldType);
+        executeTestCase(numDocs, fieldName, query, consumer, verify, true, fieldType);
+
+    }
+
+    private void executeTestCase(int numDocs,
+                                 String fieldName,
+                                 Query query,
+                                 Consumer<Document> consumer,
+                                 Consumer<InternalMissing> verify,
+                                 boolean reduced,
+                                 MappedFieldType fieldType) throws IOException {
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
-                writeIndex.accept(indexWriter);
+                Document document = new Document();
+                for (int i = 0; i < numDocs; i++) {
+                    if (frequently()) {
+                        indexWriter.commit();
+                    }
+                    consumer.accept(document);
+                    indexWriter.addDocument(document);
+                    document.clear();
+                }
             }
 
             try (IndexReader indexReader = DirectoryReader.open(directory)) {
-                final IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
-                final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]);
-                final I result;
+                IndexSearcher indexSearcher =
+                    newSearcher(indexReader, true, true);
+                MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null);
+                builder.field(fieldName);
+
+                InternalMissing missing;
                 if (reduced) {
-                    result = searchAndReduce(indexSearcher, query, builder, fieldTypesArray);
+                    missing = searchAndReduce(indexSearcher, query, builder, fieldType);
                 } else {
-                    result = search(indexSearcher, query, builder, fieldTypesArray);
+                    missing = search(indexSearcher, query, builder, fieldType);
                 }
-                verify.accept(result);
+                verify.accept(missing);
             }
         }
     }
-
-    @Override
-    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
-        return new MissingAggregationBuilder("_name", null)
-            .field(fieldName);
-    }
-
-    @Override
-    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
-        return List.of(
-            CoreValuesSourceType.NUMERIC,
-            CoreValuesSourceType.BYTES,
-            CoreValuesSourceType.GEOPOINT,
-            CoreValuesSourceType.RANGE,
-            CoreValuesSourceType.HISTOGRAM
-        );
-    }
-
-    @Override
-    protected ScriptService getMockScriptService() {
-        final Map<String, Function<Map<String, Object>, Object>> deterministicScripts = new HashMap<>();
-        deterministicScripts.put(VALUE_SCRIPT_PARAMS, vars -> {
-            final double value = ((Number) vars.get("_value")).doubleValue();
-            final long inc = ((Number) vars.get("inc")).longValue();
-            return value + inc;
-        });
-        deterministicScripts.put(VALUE_SCRIPT, vars -> {
-            final double value = ((Number) vars.get("_value")).doubleValue();
-            return value + DEFAULT_INC_PARAM;
-        });
-        deterministicScripts.put(FIELD_SCRIPT_PARAMS, vars -> {
-            final String fieldName = (String) vars.get("field");
-            final long threshold = ((Number) vars.get("threshold")).longValue();
-            return threshold(fieldName, threshold, vars);
-        });
-        deterministicScripts.put(FIELD_SCRIPT, vars -> {
-            final String fieldName = (String) vars.get("field");
-            return threshold(fieldName, DEFAULT_THRESHOLD_PARAM, vars);
-        });
-        final MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, deterministicScripts, emptyMap(), emptyMap());
-        final Map<String, ScriptEngine> engines = singletonMap(scriptEngine.getType(), scriptEngine);
-        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
-    }
-
-    private static List<Long> threshold(String fieldName, long threshold, Map<String, Object> vars) {
-        final LeafDocLookup lookup = (LeafDocLookup) vars.get("doc");
-        return lookup.get(fieldName).stream()
-            .map(value -> ((Number) value).longValue())
-            .filter(value -> value >= threshold)
-            .collect(toList());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index 04a0d71b650a9..c67850436b393 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalAggregationsTests;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.suggest.SuggestTests;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -55,8 +54,7 @@ public QuerySearchResultTests() {
 
     private static QuerySearchResult createTestInstance() throws Exception {
         ShardId shardId = new ShardId("index", "uuid", randomInt());
-        QuerySearchResult result = new QuerySearchResult(new SearchContextId("", randomLong()),
-            new SearchShardTarget("node", shardId, null, OriginalIndices.NONE));
+        QuerySearchResult result = new QuerySearchResult(randomLong(), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE));
         if (randomBoolean()) {
             result.terminatedEarly(randomBoolean());
         }
@@ -77,7 +75,7 @@ public void testSerialization() throws Exception {
         QuerySearchResult querySearchResult = createTestInstance();
         Version version = VersionUtils.randomVersion(random());
         QuerySearchResult deserialized = copyWriteable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new, version);
-        assertEquals(querySearchResult.getContextId(), deserialized.getContextId());
+        assertEquals(querySearchResult.getRequestId(), deserialized.getRequestId());
         assertNull(deserialized.getSearchShardTarget());
         assertEquals(querySearchResult.topDocs().maxScore, deserialized.topDocs().maxScore, 0f);
         assertEquals(querySearchResult.topDocs().topDocs.totalHits, deserialized.topDocs().topDocs.totalHits);
diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 404e8f92ec8f4..336a5500cfc71 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -19,12 +19,7 @@
 
 package org.elasticsearch.search.query;
 
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.English;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
@@ -33,17 +28,12 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.document.DocumentField;
-import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.analysis.CharFilterFactory;
-import org.elasticsearch.index.analysis.NormalizingCharFilterFactory;
-import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
@@ -52,14 +42,11 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.index.query.WildcardQueryBuilder;
 import org.elasticsearch.index.query.WrapperQueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.TermsLookup;
-import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
-import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
@@ -71,20 +58,16 @@
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
-import java.io.Reader;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Map;
+import java.util.Collections;
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
-import java.util.regex.Pattern;
 
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -134,7 +117,7 @@ public class SearchQueryIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(InternalSettingsPlugin.class, MockAnalysisPlugin.class);
+        return Collections.singleton(InternalSettingsPlugin.class);
     }
 
     @Override
@@ -1775,121 +1758,4 @@ public void testFieldAliasesForMetaFields() throws Exception {
        }
 
    }
-
-   /**
-    * Test that wildcard queries on keyword fields get normalized
-    */
-    public void testWildcardQueryNormalizationOnKeywordField() {
-       assertAcked(prepareCreate("test")
-               .setSettings(Settings.builder()
-                       .put("index.analysis.normalizer.lowercase_normalizer.type", "custom")
-                       .putList("index.analysis.normalizer.lowercase_normalizer.filter", "lowercase")
-                       .build())
-                .setMapping("field1", "type=keyword,normalizer=lowercase_normalizer"));
-       client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
-       refresh();
-
-        {
-            WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
-            SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
-            assertHitCount(searchResponse, 1L);
-
-            wildCardQuery = wildcardQuery("field1", "bb*");
-            searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
-            assertHitCount(searchResponse, 1L);
-        }
-   }
-
-    /**
-     * Test that wildcard queries on text fields get normalized
-     */
-     public void testWildcardQueryNormalizationOnTextField() {
-        assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
-                        .put("index.analysis.analyzer.lowercase_analyzer.type", "custom")
-                        .put("index.analysis.analyzer.lowercase_analyzer.tokenizer", "standard")
-                        .putList("index.analysis.analyzer.lowercase_analyzer.filter", "lowercase")
-                        .build())
-                 .setMapping("field1", "type=text,analyzer=lowercase_analyzer"));
-        client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
-        refresh();
-
-         {
-             WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
-             SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
-             assertHitCount(searchResponse, 1L);
-
-             wildCardQuery = wildcardQuery("field1", "bb*");
-             searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
-             assertHitCount(searchResponse, 1L);
-         }
-    }
-
-    /**
-     * Reserved characters should be excluded when the normalization is applied for keyword fields.
-     * See https://github.com/elastic/elasticsearch/issues/46300 for details.
-     */
-    public void testWildcardQueryNormalizationKeywordSpecialCharacters() {
-        assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder().put("index.analysis.char_filter.no_wildcard.type", "mock_pattern_replace")
-                        .put("index.analysis.normalizer.no_wildcard.type", "custom")
-                        .put("index.analysis.normalizer.no_wildcard.char_filter", "no_wildcard").build())
-                .setMapping("field", "type=keyword,normalizer=no_wildcard"));
-        client().prepareIndex("test").setId("1").setSource("field", "label-1").get();
-        refresh();
-
-        WildcardQueryBuilder wildCardQuery = wildcardQuery("field", "la*");
-        SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
-        assertHitCount(searchResponse, 1L);
-
-        wildCardQuery = wildcardQuery("field", "la*el-?");
-        searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
-        assertHitCount(searchResponse, 1L);
-    }
-
-    public static class MockAnalysisPlugin extends Plugin implements AnalysisPlugin {
-
-        @Override
-        public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
-            return singletonMap("mock_pattern_replace", (indexSettings, env, name, settings) -> {
-                class Factory implements NormalizingCharFilterFactory {
-
-                    private final Pattern pattern = Regex.compile("[\\*\\?]", null);
-
-                    @Override
-                    public String name() {
-                        return name;
-                    }
-
-                    @Override
-                    public Reader create(Reader reader) {
-                        return new PatternReplaceCharFilter(pattern, "", reader);
-                    }
-                }
-                return new Factory();
-            });
-        }
-
-        @Override
-        public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
-            return singletonMap("keyword", (indexSettings, environment, name, settings) -> TokenizerFactory.newFactory(name,
-                    () -> new MockTokenizer(MockTokenizer.KEYWORD, false)));
-        }
-    }
-
-    /**
-     * Test correct handling {@link SpanBooleanQueryRewriteWithMaxClause#rewrite(IndexReader, MultiTermQuery)}. That rewrite method is e.g.
-     * set for fuzzy queries with "constant_score" rewrite nested inside a `span_multi` query and would cause NPEs due to an unset
-     * {@link AttributeSource}.
-     */
-    public void testIssueFuzzyInsideSpanMulti() {
-        createIndex("test");
-        client().prepareIndex("test").setId("1").setSource("field", "foobarbaz").get();
-        ensureGreen();
-        refresh();
-
-        BoolQueryBuilder query = boolQuery().filter(spanMultiTermQueryBuilder(fuzzyQuery("field", "foobarbiz").rewrite("constant_score")));
-        SearchResponse response = client().prepareSearch("test").setQuery(query).get();
-        assertHitCount(response, 1);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index 6e217a97ddf42..e11f74be0bab2 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -21,11 +21,9 @@
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.search.ClearScrollResponse;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -36,7 +34,6 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.rest.RestStatus;
@@ -44,7 +41,6 @@
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.junit.After;
 
@@ -659,42 +655,6 @@ public void testScrollRewrittenToMatchNoDocs() {
         }
     }
 
-    public void testRestartDataNodesDuringScrollSearch() throws Exception {
-        final String dataNode = internalCluster().startDataOnlyNode();
-        createIndex("demo", Settings.builder()
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.routing.allocation.include._name", dataNode)
-            .build());
-        createIndex("prod", Settings.builder()
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.routing.allocation.include._name", dataNode)
-            .build());
-        int numDocs = randomIntBetween(10, 100);
-        for (int i = 0; i < numDocs; i++) {
-            index("demo", "demo-" + i, Map.of());
-            index("prod", "prod-" + i, Map.of());
-        }
-        client().admin().indices().prepareRefresh().get();
-        SearchResponse respFromDemoIndex = client().prepareSearch("demo")
-            .setSize(randomIntBetween(1, 10))
-            .setQuery(new MatchAllQueryBuilder()).setScroll(TimeValue.timeValueMinutes(5)).get();
-
-        internalCluster().restartNode(dataNode, new InternalTestCluster.RestartCallback());
-        ensureGreen("demo", "prod");
-        SearchResponse respFromProdIndex = client().prepareSearch("prod")
-            .setSize(randomIntBetween(1, 10))
-            .setQuery(new MatchAllQueryBuilder()).setScroll(TimeValue.timeValueMinutes(5)).get();
-        assertNoFailures(respFromProdIndex);
-        SearchPhaseExecutionException error = expectThrows(SearchPhaseExecutionException.class,
-            () -> client().prepareSearchScroll(respFromDemoIndex.getScrollId()).get());
-        for (ShardSearchFailure shardSearchFailure : error.shardFailures()) {
-            assertThat(shardSearchFailure.getCause().getMessage(), containsString("No search context found for id [1]"));
-        }
-        client().prepareSearchScroll(respFromProdIndex.getScrollId()).get();
-    }
-
     private void assertToXContentResponse(ClearScrollResponse response, boolean succeed, int numFreed) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
index ee74c37fbd401..44fe0d4dd5c58 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
@@ -66,6 +66,7 @@ public void testRestoreSnapshotRequestParsing() throws IOException {
         builder.field("rename_replacement", "rename-to");
         boolean partial = randomBoolean();
         builder.field("partial", partial);
+        builder.startObject("settings").field("set1", "val1").endObject();
         builder.startObject("index_settings").field("set1", "val2").endObject();
         if (randomBoolean()) {
             builder.field("ignore_index_settings", "set2,set3");
@@ -90,6 +91,7 @@ public void testRestoreSnapshotRequestParsing() throws IOException {
         assertEquals("rename-from", request.renamePattern());
         assertEquals("rename-to", request.renameReplacement());
         assertEquals(partial, request.partial());
+        assertEquals("val1", request.settings().get("set1"));
         assertArrayEquals(request.ignoreIndexSettings(), new String[]{"set2", "set3"});
         boolean expectedIgnoreAvailable = includeIgnoreUnavailable
             ? indicesOptions.ignoreUnavailable()
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index 2e3102cb7e926..f8a4fe4b9429b 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -417,8 +417,8 @@ public void testRenderConnectionInfoXContent() throws IOException {
                 "\"num_nodes_connected\":2,\"max_connections_per_cluster\":3,\"initial_connect_timeout\":\"30m\"," +
                 "\"skip_unavailable\":true}}", Strings.toString(builder));
         } else {
-            assertEquals("{\"test_cluster\":{\"connected\":true,\"mode\":\"proxy\",\"proxy_address\":\"seed:1\"," +
-                "\"num_proxy_sockets_connected\":16,\"max_proxy_socket_connections\":18,\"initial_connect_timeout\":\"30m\"," +
+            assertEquals("{\"test_cluster\":{\"connected\":true,\"mode\":\"proxy\",\"address\":\"seed:1\"," +
+                "\"num_sockets_connected\":16,\"max_socket_connections\":18,\"initial_connect_timeout\":\"30m\"," +
                 "\"skip_unavailable\":true}}", Strings.toString(builder));
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 75e577680274a..21078a085b017 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -728,12 +728,6 @@ public static String randomRealisticUnicodeOfCodepointLength(int codePoints) {
         return RandomizedTest.randomRealisticUnicodeOfCodepointLength(codePoints);
     }
 
-    /**
-     * @param maxArraySize The maximum number of elements in the random array
-     * @param stringSize The length of each String in the array
-     * @param allowNull Whether the returned array may be null
-     * @param allowEmpty Whether the returned array may be empty (have zero elements)
-     */
     public static String[] generateRandomStringArray(int maxArraySize, int stringSize, boolean allowNull, boolean allowEmpty) {
         if (allowNull && random().nextBoolean()) {
             return null;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index 12f06bd6a9deb..e7b5e937c0c02 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -52,7 +52,6 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -133,8 +132,8 @@ public Query buildFilteredQuery(Query query) {
     }
 
     @Override
-    public SearchContextId id() {
-        return new SearchContextId("", 0);
+    public long id() {
+        return 0;
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
index 1449252d024ac..7e80810d7dd27 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
@@ -58,13 +58,6 @@ public static Map<String, Object> convertToMap(ToXContent part) throws IOExcepti
         return XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
     }
 
-    public static BytesReference convertToXContent(Map<String, ?> map, XContentType xContentType) throws IOException {
-        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
-            builder.map(map);
-            return BytesReference.bytes(builder);
-        }
-    }
-
 
     /**
      * Compares two maps generated from XContentObjects. The order of elements in arrays is ignored.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 445fa98af651f..2ecac6daac8f4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -1126,7 +1126,6 @@ protected static boolean isXPackTemplate(String name) {
         case ".logstash-management":
         case "security_audit_log":
         case ".slm-history":
-        case ".async-search":
             return true;
         default:
             return false;
diff --git a/x-pack/docs/en/security/authentication/custom-realm.asciidoc b/x-pack/docs/en/security/authentication/custom-realm.asciidoc
index 99f124b27397e..857444c0836d7 100644
--- a/x-pack/docs/en/security/authentication/custom-realm.asciidoc
+++ b/x-pack/docs/en/security/authentication/custom-realm.asciidoc
@@ -11,7 +11,8 @@ extension as part of an ordinary elasticsearch plugin.
 ==== Implementing a custom realm
 
 Sample code that illustrates the structure and implementation of a custom realm
-is provided in https://github.com/elastic/elasticsearch/tree/{branch}/x-pack/qa/security-example-spi-extension. You can use this code as a starting point for creating your
+is provided in the https://github.com/elastic/shield-custom-realm-example[custom-realm-example]
+repository on GitHub. You can use this code as a starting point for creating your
 own realm.
 
 To create a custom realm, you need to:
@@ -85,7 +86,7 @@ bin/elasticsearch-plugin install file:///<path>/my-realm-1.0.zip
 
 . Add a realm configuration of the appropriate realm type to `elasticsearch.yml`
 under the `xpack.security.authc.realms` namespace.
-You must define your realm within the namespace that matches the type defined
+You must define your realm within the namespace that matchesto the type defined
 by the extension.
 The options you can set depend on the settings exposed by the custom realm.
 At a minimum, you must explicitly set the `order` attribute to control the
diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
index 1cf13fbd2a15d..c100ef161c755 100644
--- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc
+++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
@@ -879,7 +879,7 @@ an {es} access token and a refresh token. The access token is used as credential
 for subsequent calls to {es}. The refresh token enables the user to get new {es}
 access tokens after the current one expires.
 
-[[saml-no-kibana-realm]]
+[[saml-no-kibana-realm]]]
 ==== SAML realm
 
 You must create a SAML realm and configure it accordingly
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java
index eb7474b7bffd6..610cf89641188 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -21,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -94,7 +94,7 @@ public Object getProperty(List<String> path) {
             return null;
         }
         assert topMetrics.size() == 1 : "property paths should only resolve against top metrics with size == 1.";
-        return topMetrics.get(0).metricValues.get(index).numberValue();
+        return topMetrics.get(0).metricValues[index];
     }
 
     @Override
@@ -161,14 +161,13 @@ public boolean equals(Object obj) {
     public double value(String name) {
         int index = metricNames.indexOf(name);
         if (index < 0) {
-            throw new IllegalArgumentException("unknown metric [" + name + "]");
+            throw new IllegalArgumentException("unknown metric [" + name + "]");            
         }
         if (topMetrics.isEmpty()) {
             return Double.NaN;
         }
         assert topMetrics.size() == 1 : "property paths should only resolve against top metrics with size == 1.";
-        // TODO it'd probably be nicer to have "compareTo" instead of assuming a double.
-        return topMetrics.get(0).metricValues.get(index).numberValue().doubleValue();
+        return topMetrics.get(0).metricValues[index];
     }
 
     SortOrder getSortOrder() {
@@ -207,9 +206,9 @@ TopMetric topMetric() {
     static class TopMetric implements Writeable, Comparable<TopMetric> {
         private final DocValueFormat sortFormat;
         private final SortValue sortValue;
-        private final List<MetricValue> metricValues;
+        private final double[] metricValues;
 
-        TopMetric(DocValueFormat sortFormat, SortValue sortValue, List<MetricValue> metricValues) {
+        TopMetric(DocValueFormat sortFormat, SortValue sortValue, double[] metricValues) {
             this.sortFormat = sortFormat;
             this.sortValue = sortValue;
             this.metricValues = metricValues;
@@ -218,14 +217,14 @@ static class TopMetric implements Writeable, Comparable<TopMetric> {
         TopMetric(StreamInput in) throws IOException {
             sortFormat = in.readNamedWriteable(DocValueFormat.class);
             sortValue = in.readNamedWriteable(SortValue.class);
-            metricValues = in.readList(s -> s.readOptionalWriteable(MetricValue::new));
+            metricValues = in.readDoubleArray();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeNamedWriteable(sortFormat);
             out.writeNamedWriteable(sortValue);
-            out.writeCollection(metricValues, StreamOutput::writeOptionalWriteable);
+            out.writeDoubleArray(metricValues);
         }
 
         DocValueFormat getSortFormat() {
@@ -236,7 +235,7 @@ SortValue getSortValue() {
             return sortValue;
         }
 
-        List<MetricValue> getMetricValues() {
+        double[] getMetricValues() {
             return metricValues;
         }
 
@@ -247,13 +246,9 @@ public XContentBuilder toXContent(XContentBuilder builder, List<String> metricNa
                 sortValue.toXContent(builder, sortFormat);
                 builder.endArray();
                 builder.startObject(METRIC_FIELD.getPreferredName());
-                for (int i = 0; i < metricValues.size(); i++) {
-                    MetricValue value = metricValues.get(i);
-                    builder.field(metricNames.get(i));
-                    if (value == null) {
-                        builder.nullValue();
-                    } else {
-                        value.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                {
+                    for (int i = 0; i < metricValues.length; i++) {
+                        builder.field(metricNames.get(i), Double.isNaN(metricValues[i]) ? null : metricValues[i]);
                     }
                 }
                 builder.endObject();
@@ -274,79 +269,17 @@ public boolean equals(Object obj) {
             TopMetric other = (TopMetric) obj;
             return sortFormat.equals(other.sortFormat)
                     && sortValue.equals(other.sortValue)
-                    && metricValues.equals(other.metricValues);
+                    && Arrays.equals(metricValues, other.metricValues);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(sortFormat, sortValue, metricValues);
+            return Objects.hash(sortFormat, sortValue, Arrays.hashCode(metricValues));
         }
 
         @Override
         public String toString() {
-            return "TopMetric[" + sortFormat + "," + sortValue + "," + metricValues + "]"; 
-        }
-    }
-
-    static class MetricValue implements Writeable, ToXContent {
-        private final DocValueFormat format;
-        /**
-         * It is odd to have a "SortValue" be part of a MetricValue but it is
-         * a very convenient way to send a type-aware thing across the
-         * wire though. So here we are.
-         */
-        private final SortValue value;
-
-        MetricValue(DocValueFormat format, SortValue value) {
-            this.format = format;
-            this.value = value;
-        }
-
-        DocValueFormat getFormat() {
-            return format;
-        }
-
-        SortValue getValue() {
-            return value;
-        }
-
-        MetricValue(StreamInput in) throws IOException {
-            format = in.readNamedWriteable(DocValueFormat.class);
-            value = in.readNamedWriteable(SortValue.class);
-        }
-
-        Number numberValue() {
-            return value.numberValue();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeNamedWriteable(format);
-            out.writeNamedWriteable(value);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return value.toXContent(builder, format);
-        }
-
-        @Override
-        public String toString() {
-            return format + "," + value;
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (obj == null || obj.getClass() != getClass()) {
-                return false;
-            }
-            MetricValue other = (MetricValue) obj;
-            return format.equals(other.format) && value.equals(other.value);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(format, value);
+            return "TopMetric[" + sortFormat + "," + sortValue + "," + Arrays.toString(metricValues) + "]"; 
         }
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java
index e25a80b31e1bd..300f583050837 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java
@@ -7,16 +7,14 @@
 package org.elasticsearch.xpack.analytics.topmetrics;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.BitArray;
 import org.elasticsearch.common.util.DoubleArray;
-import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -28,17 +26,11 @@
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.BucketedSort;
 import org.elasticsearch.search.sort.SortBuilder;
-import org.elasticsearch.search.sort.SortValue;
-import org.elasticsearch.xpack.analytics.topmetrics.InternalTopMetrics.MetricValue;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import static java.util.stream.Collectors.toList;
-
 /**
  * Collects the {@code top_metrics} aggregation, which functions like a memory
  * efficient but limited version of the {@code top_hits} aggregation. Amortized,
@@ -60,16 +52,17 @@ class TopMetricsAggregator extends NumericMetricsAggregator.MultiValue {
 
     TopMetricsAggregator(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData, int size,
-            SortBuilder<?> sort, List<MetricSource> metricSources) throws IOException {
+            SortBuilder<?> sort, List<String> metricNames, List<ValuesSource.Numeric> metricValuesSources) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.size = size;
-        metrics = new Metrics(size, context.getQueryShardContext().bigArrays(), metricSources);
+        assert metricNames.size() == metricValuesSources.size();
+        metrics = new Metrics(size, context.getQueryShardContext(), metricNames, metricValuesSources);
         /*
          * If we're only collecting a single value then only provided *that*
          * value to the sort so that swaps and loads are just a little faster
          * in that *very* common case.
          */
-        BucketedSort.ExtraData values = metricSources.size() == 1 ? metrics.values[0] : metrics;
+        BucketedSort.ExtraData values = metricValuesSources.size() == 1 ? metrics.values[0] : metrics;
         this.sort = sort.buildBucketedSort(context.getQueryShardContext(), size, values);
     }
 
@@ -78,12 +71,7 @@ public boolean hasMetric(String name) {
         if (size != 1) {
             throw new IllegalArgumentException("[top_metrics] can only the be target if [size] is [1] but was [" + size + "]");
         }
-        for (MetricValues values : metrics.values) {
-            if (values.name().equals(name)) {
-                return true;
-            }
-        }
-        return false;
+        return metrics.names.contains(name);
     }
 
     @Override
@@ -127,12 +115,12 @@ public void setScorer(Scorable s) throws IOException {
     public InternalAggregation buildAggregation(long bucket) throws IOException {
         List<InternalTopMetrics.TopMetric> topMetrics = sort.getValues(bucket, metrics.resultBuilder(sort.getFormat()));
         assert topMetrics.size() <= size;
-        return new InternalTopMetrics(name, sort.getOrder(), metrics.names(), size, topMetrics, pipelineAggregators(), metaData());
+        return new InternalTopMetrics(name, sort.getOrder(), metrics.names, size, topMetrics, pipelineAggregators(), metaData());
     }
 
     @Override
     public InternalTopMetrics buildEmptyAggregation() {
-        return InternalTopMetrics.buildEmptyAggregation(name, metrics.names(), pipelineAggregators(), metaData());
+        return InternalTopMetrics.buildEmptyAggregation(name, metrics.names, pipelineAggregators(), metaData());
     }
 
     @Override
@@ -140,49 +128,21 @@ public void doClose() {
         Releasables.close(sort, metrics);
     }
 
-    /**
-     * Information about each metric that this {@link Aggregator} uses to
-     * load and format metric values.
-     */
-    static class MetricSource {
-        private final String name;
-        private final DocValueFormat format;
-        private final ValuesSource.Numeric valuesSource;
-
-        MetricSource(String name, DocValueFormat format, ValuesSource.Numeric valuesSource) {
-            this.name = name;
-            this.format = format;
-            this.valuesSource = valuesSource;
-        }
-
-        String getName() {
-            return name;
-        }
-
-        DocValueFormat getFormat() {
-            return format;
-        }
-    }
-
-    static class Metrics implements BucketedSort.ExtraData, Releasable {
+    private static class Metrics implements BucketedSort.ExtraData, Releasable {
+        private final List<String> names;
         private final MetricValues[] values;
 
-        Metrics(int size, BigArrays bigArrays, List<MetricSource> sources) {
-            values = new MetricValues[sources.size()];
+        Metrics(int size, QueryShardContext ctx, List<String> names, List<ValuesSource.Numeric> valuesSources) {
+            this.names = names; 
+            values = new MetricValues[valuesSources.size()];
             int i = 0;
-            for (MetricSource source : sources) {
-                values[i++] = valuesFor(size, bigArrays, source);
-            }
-        }
-
-        private static MetricValues valuesFor(int size, BigArrays bigArrays, MetricSource source) {
-            if (source.valuesSource == null) {
-                return new AlwaysNullMetricValues(source);
-            }
-            if (source.valuesSource.isFloatingPoint()) {
-                return new DoubleMetricValues(size, bigArrays, source);
+            for (ValuesSource.Numeric valuesSource : valuesSources) {
+                if (valuesSource == null) {
+                    values[i++] = new MissingMetricValues();
+                    continue;
+                }
+                values[i++] = new CollectMetricValues(size, ctx.bigArrays(), valuesSource);
             }
-            return new LongMetricValues(size, bigArrays, source);
         }
 
         boolean needsScores() {
@@ -195,28 +155,23 @@ boolean needsScores() {
         }
 
         double metric(String name, long index) {
-            for (MetricValues value : values) {
-                if (value.name().equals(name)) {
-                    return value.doubleValue(index);
-                }
+            int valueIndex = names.indexOf(name);
+            if (valueIndex < 0) {
+                throw new IllegalArgumentException("[" + name + "] not found");
             }
-            throw new IllegalArgumentException("[" + name + "] not found");
+            return values[valueIndex].value(index);
         }
 
         BucketedSort.ResultBuilder<InternalTopMetrics.TopMetric> resultBuilder(DocValueFormat sortFormat) {
             return (index, sortValue) -> {
-                List<InternalTopMetrics.MetricValue> result = new ArrayList<>(values.length);
+                double[] result = new double[values.length];
                 for (int i = 0; i < values.length; i++) {
-                    result.add(values[i].metricValue(index));
+                    result[i] = values[i].value(index);
                 }
                 return new InternalTopMetrics.TopMetric(sortFormat, sortValue, result);
             };
         }
 
-        List<String> names() {
-            return Arrays.stream(values).map(MetricValues::name).collect(toList());
-        }
-
         @Override
         public void swap(long lhs, long rhs) {
             for (int i = 0; i < values.length; i++) {
@@ -243,60 +198,30 @@ public void close() {
         }
     }
 
-    private abstract static class MetricValues implements BucketedSort.ExtraData, Releasable {
-        protected final MetricSource source;
-
-        MetricValues(MetricSource source) {
-            this.source = source;
-        }
-
-        final String name() {
-            return source.name;
-        }
-
-        abstract boolean needsScores();
-        abstract double doubleValue(long index);
-        abstract InternalTopMetrics.MetricValue metricValue(long index);
-    }
-
-    private abstract static class CollectingMetricValues extends MetricValues {
-        protected final BigArrays bigArrays;
-
-        CollectingMetricValues(BigArrays bigArrays, MetricSource source) {
-            super(source);
-            this.bigArrays = bigArrays;
-        }
-
-        @Override
-        public final boolean needsScores() {
-            return source.valuesSource.needsScores();
-        }
+    private interface MetricValues extends BucketedSort.ExtraData, Releasable {
+        boolean needsScores();
+        double value(long index);
     }
+    private static class CollectMetricValues implements MetricValues {
+        private final BigArrays bigArrays;
+        private final ValuesSource.Numeric metricValueSource;
 
-    /**
-     * Loads metrics for floating point numbers.
-     */
-    static class DoubleMetricValues extends CollectingMetricValues {
         private DoubleArray values;
 
-        DoubleMetricValues(int size, BigArrays bigArrays, MetricSource source) {
-            super(bigArrays, source);
+        CollectMetricValues(int size, BigArrays bigArrays, ValuesSource.Numeric metricValueSource) {
+            this.bigArrays = bigArrays;
+            this.metricValueSource = metricValueSource;
             values = bigArrays.newDoubleArray(size, false);
         }
 
         @Override
-        public double doubleValue(long index) {
-            return values.get(index);
+        public boolean needsScores() {
+            return metricValueSource.needsScores();
         }
 
         @Override
-        public MetricValue metricValue(long index) {
-            double value = values.get(index);
-            if (Double.isNaN(value)) {
-                // Use NaN as a sentinel for "missing"
-                return null;
-            }
-            return new MetricValue(source.format, SortValue.from(value));
+        public double value(long index) {
+            return values.get(index);
         }
 
         @Override
@@ -309,13 +234,12 @@ public void swap(long lhs, long rhs) {
         @Override
         public Loader loader(LeafReaderContext ctx) throws IOException {
             // TODO allow configuration of value mode
-            NumericDoubleValues metricValues = MultiValueMode.AVG.select(source.valuesSource.doubleValues(ctx));
+            NumericDoubleValues metricValues = MultiValueMode.AVG.select(metricValueSource.doubleValues(ctx));
             return (index, doc) -> {
                 if (index >= values.size()) {
                     values = bigArrays.grow(values, index + 1);
                 }
-                // Use NaN as a sentinel for "missing"
-                double metricValue = metricValues.advanceExact(doc) ? metricValues.doubleValue() : Double.NaN;
+                double metricValue = metricValues.advanceExact(doc) ? metricValues.doubleValue() : Double.NaN; 
                 values.set(index, metricValue);
             };
         }
@@ -325,93 +249,12 @@ public void close() {
             values.close();
         }
     }
-
-    /**
-     * Loads metrics for whole numbers.
-     */
-    static class LongMetricValues extends CollectingMetricValues {
-        /**
-         * Tracks "missing" values in a {@link BitArray}. Unlike
-         * {@link DoubleMetricValues}, we there isn't a sentinel value
-         * that we can steel from the longs to represent missing that
-         * won't lead to more trouble than it is worth. So we track
-         * "missing" values explicitly.
-         */
-        private final MissingHelper empty;
-        private LongArray values;
-
-        LongMetricValues(int size, BigArrays bigArrays, MetricSource source) {
-            super(bigArrays, source);
-            empty = new MissingHelper(bigArrays);
-            values = bigArrays.newLongArray(size, false);
-        }
-
-        @Override
-        public double doubleValue(long index) {
-            if (empty.isEmpty(index)) {
-                return Double.NaN;
-            }
-            return values.get(index);
-        }
-
-        @Override
-        public MetricValue metricValue(long index) {
-            if (empty.isEmpty(index)) {
-                return null;
-            }
-            return new MetricValue(source.format, SortValue.from(values.get(index)));
-        }
-
-        @Override
-        public void swap(long lhs, long rhs) {
-            long tmp = values.get(lhs);
-            values.set(lhs, values.get(rhs));
-            values.set(rhs, tmp);
-            empty.swap(lhs, rhs);
-        }
-
-        @Override
-        public Loader loader(LeafReaderContext ctx) throws IOException {
-            // TODO allow configuration of value mode
-            NumericDocValues metricValues = MultiValueMode.AVG.select(source.valuesSource.longValues(ctx));
-            return (index, doc) -> {
-                if (false == metricValues.advanceExact(doc)) {
-                    empty.markMissing(index);
-                    return;
-                }
-                if (index >= values.size()) {
-                    values = bigArrays.grow(values, index + 1);
-                }
-                values.set(index, metricValues.longValue());
-                empty.markNotMissing(index);
-            };
-        }
-
-        @Override
-        public void close() {
-            Releasables.close(values, empty);
-        }
-    }
-
-    /**
-     * {@linkplain MetricValues} implementation for unmapped fields
-     * that always returns {@code null} or {@code NaN}.
-     */
-    static class AlwaysNullMetricValues extends MetricValues {
-        AlwaysNullMetricValues(MetricSource source) {
-            super(source);
-        }
-
+    private static class MissingMetricValues implements MetricValues {
         @Override
-        public double doubleValue(long index) {
+        public double value(long index) {
             return Double.NaN;
         }
 
-        @Override
-        public MetricValue metricValue(long index) {
-            return null;
-        }
-
         @Override
         public boolean needsScores() {
             return false;
@@ -425,76 +268,8 @@ public Loader loader(LeafReaderContext ctx) throws IOException {
             return (index, doc) -> {};
         }
 
-        @Override
-        public void close() {}
-    }
-
-    /**
-     * Helps {@link LongMetricValues} track "empty" slots. It attempts to have
-     * very low CPU overhead and no memory overhead when there *aren't* empty
-     * values.
-     */
-    private static class MissingHelper implements Releasable {
-        private final BigArrays bigArrays;
-        private BitArray tracker;
-
-        MissingHelper(BigArrays bigArrays) {
-            this.bigArrays = bigArrays;
-        }
-
-        void markMissing(long index) {
-            int i = asInt(index);
-            if (tracker == null) {
-                tracker = new BitArray(i, bigArrays);
-            }
-            tracker.set(i);
-        }
-
-        void markNotMissing(long index) {
-            if (tracker == null) {
-                return;
-            }
-            tracker.clear(asInt(index));
-        }
-
-        void swap(long lhs, long rhs) {
-            if (tracker == null) {
-                return;
-            }
-            int l = asInt(lhs);
-            int r = asInt(rhs);
-            boolean backup = tracker.get(l);
-            if (tracker.get(r)) {
-                tracker.set(l);
-            } else {
-                tracker.clear(l);
-            }
-            if (backup) {
-                tracker.set(r);
-            } else {
-                tracker.clear(r);
-            }
-        }
-
-        boolean isEmpty(long index) {
-            if (tracker == null) {
-                return false;
-            }
-            return tracker.get(asInt(index));
-        }
-
         @Override
         public void close() {
-            if (tracker != null) {
-                tracker.close();
-            }
-        }
-
-        private int asInt(long index) {
-            if (index > Integer.MAX_VALUE) {
-                throw new IllegalArgumentException("top_metrics can't track more than " + Integer.MAX_VALUE + " values.");
-            }
-            return (int) index;
         }
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java
index 2c2599e450a97..be63f886dede5 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java
@@ -57,14 +57,14 @@ protected TopMetricsAggregator createInternal(SearchContext searchContext, Aggre
                     + "]. This limit can be set by changing the [" + MAX_BUCKET_SIZE.getKey()
                     + "] index level setting.");
         }
-        List<TopMetricsAggregator.MetricSource> metricSources = metricFields.stream().map(config -> {
+        List<String> metricNames = metricFields.stream().map(MultiValuesSourceFieldConfig::getFieldName).collect(toList());
+        List<ValuesSource.Numeric> metricValuesSources = metricFields.stream().map(config -> {
                     ValuesSourceConfig<ValuesSource.Numeric> resolved = ValuesSourceConfig.resolve(
                             searchContext.getQueryShardContext(), ValueType.NUMERIC,
                             config.getFieldName(), config.getScript(), config.getMissing(), config.getTimeZone(), null);
-                    return new TopMetricsAggregator.MetricSource(config.getFieldName(), resolved.format(),
-                            resolved.toValuesSource(searchContext.getQueryShardContext()));
+                    return resolved.toValuesSource(searchContext.getQueryShardContext());
                 }).collect(toList());
         return new TopMetricsAggregator(name, searchContext, parent, pipelineAggregators, metaData, size,
-                sortBuilders.get(0), metricSources);
+                sortBuilders.get(0), metricNames, metricValuesSources);
     }
 }
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java
index eae410e015dfa..668b6a3aea8e8 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java
@@ -96,9 +96,7 @@ private InternalTopMetrics buildFilled(SortOrder sortOrder, int size, InternalTo
 
     private InternalTopMetrics.TopMetric top(SortValue sortValue, double metricValue) {
         DocValueFormat sortFormat = randomFrom(DocValueFormat.RAW, DocValueFormat.BINARY, DocValueFormat.BOOLEAN, DocValueFormat.IP);
-        DocValueFormat metricFormat = randomFrom(DocValueFormat.RAW, DocValueFormat.BINARY, DocValueFormat.BOOLEAN, DocValueFormat.IP);
-        InternalTopMetrics.MetricValue realMetricValue = new InternalTopMetrics.MetricValue(metricFormat, SortValue.from(metricValue));
-        return new InternalTopMetrics.TopMetric(sortFormat, sortValue, singletonList(realMetricValue));
+        return new InternalTopMetrics.TopMetric(sortFormat, sortValue, new double[] {metricValue});
     }
 
     private InternalTopMetrics reduce(InternalTopMetrics... results) {
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
index 64c041604fc5d..c8bda2dabec0e 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
@@ -31,15 +31,13 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
-import java.util.function.Supplier;
 import java.util.stream.IntStream;
 
 import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasSize;
 
 public class InternalTopMetricsTests extends InternalAggregationTestCase<InternalTopMetrics> {
@@ -50,11 +48,7 @@ public class InternalTopMetricsTests extends InternalAggregationTestCase<Interna
      * would fail because the instances that it attempts to reduce don't
      * have their results in the same order.
      */
-    private final SortOrder sortOrder = randomFrom(SortOrder.values());
-    private final InternalTopMetrics.MetricValue metricOneDouble =
-            new InternalTopMetrics.MetricValue(DocValueFormat.RAW, SortValue.from(1.0));
-    private final InternalTopMetrics.MetricValue metricOneLong =
-            new InternalTopMetrics.MetricValue(DocValueFormat.RAW, SortValue.from(1));
+    private SortOrder sortOrder = randomFrom(SortOrder.values());
 
     public void testEmptyIsNotMapped() {
         InternalTopMetrics empty = InternalTopMetrics.buildEmptyAggregation(
@@ -68,9 +62,8 @@ public void testNonEmptyIsMapped() {
     }
 
     public void testToXContentDoubleSortValue() throws IOException {
-        List<InternalTopMetrics.TopMetric> top = singletonList(
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricOneDouble)));
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1,
+                List.of(new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {1.0})), emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -88,11 +81,12 @@ public void testToXContentDoubleSortValue() throws IOException {
                 "}"));
     }
 
-    public void testToXContentDateSortValue() throws IOException {
+    public void testToXConentDateSortValue() throws IOException {
+        DocValueFormat sortFormat = new DocValueFormat.DateTime(DateFormatter.forPattern("strict_date_time"), ZoneId.of("UTC"),
+                DateFieldMapper.Resolution.MILLISECONDS);
         SortValue sortValue = SortValue.from(ZonedDateTime.parse("2007-12-03T10:15:30Z").toInstant().toEpochMilli());
-        List<InternalTopMetrics.TopMetric> top = singletonList(new InternalTopMetrics.TopMetric(
-                strictDateTime(), sortValue, singletonList(metricOneDouble)));
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1,
+                List.of(new InternalTopMetrics.TopMetric(sortFormat, sortValue, new double[] {1.0})), emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -110,54 +104,10 @@ public void testToXContentDateSortValue() throws IOException {
                 "}"));
     }
 
-    public void testToXContentLongMetricValue() throws IOException {
-        List<InternalTopMetrics.TopMetric> top = singletonList(
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricOneLong)));
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
-        assertThat(Strings.toString(tm, true, true), equalTo(
-                "{\n" +
-                "  \"test\" : {\n" +
-                "    \"top\" : [\n" +
-                "      {\n" +
-                "        \"sort\" : [\n" +
-                "          1.0\n" +
-                "        ],\n" +
-                "        \"metrics\" : {\n" +
-                "          \"test\" : 1\n" +
-                "        }\n" +
-                "      }\n" +
-                "    ]\n" +
-                "  }\n" +
-                "}"));
-    }
-
-    public void testToXContentDateMetricValue() throws IOException {
-        InternalTopMetrics.MetricValue metricValue = new InternalTopMetrics.MetricValue(
-                strictDateTime(), SortValue.from(ZonedDateTime.parse("2007-12-03T10:15:30Z").toInstant().toEpochMilli()));
-        List<InternalTopMetrics.TopMetric> top = singletonList(
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricValue)));
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
-        assertThat(Strings.toString(tm, true, true), equalTo(
-                "{\n" +
-                "  \"test\" : {\n" +
-                "    \"top\" : [\n" +
-                "      {\n" +
-                "        \"sort\" : [\n" +
-                "          1.0\n" +
-                "        ],\n" +
-                "        \"metrics\" : {\n" +
-                "          \"test\" : \"2007-12-03T10:15:30.000Z\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    ]\n" +
-                "  }\n" +
-                "}"));
-    }
-
     public void testToXContentManyMetrics() throws IOException {
-        List<InternalTopMetrics.TopMetric> top = singletonList(new InternalTopMetrics.TopMetric(
-                DocValueFormat.RAW, SortValue.from(1.0), List.of(metricOneDouble, metricOneLong, metricOneDouble)));
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, List.of("foo", "bar", "baz"), 1, top, emptyList(), null);
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, List.of("foo", "bar", "baz"), 1,
+                List.of(new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {1.0, 2.0, 3.0})),
+                emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -168,8 +118,8 @@ public void testToXContentManyMetrics() throws IOException {
                 "        ],\n" +
                 "        \"metrics\" : {\n" +
                 "          \"foo\" : 1.0,\n" +
-                "          \"bar\" : 1,\n" +
-                "          \"baz\" : 1.0\n" +
+                "          \"bar\" : 2.0,\n" +
+                "          \"baz\" : 3.0\n" +
                 "        }\n" +
                 "      }\n" +
                 "    ]\n" +
@@ -178,10 +128,11 @@ public void testToXContentManyMetrics() throws IOException {
     }
 
     public void testToXContentManyTopMetrics() throws IOException {
-        List<InternalTopMetrics.TopMetric> top = List.of(
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricOneDouble)),
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(2.0), singletonList(metricOneLong)));
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 2, top, emptyList(), null);
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 2,
+                List.of(
+                    new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {1.0}),
+                    new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(2.0), new double[] {2.0})),
+                emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -199,7 +150,7 @@ public void testToXContentManyTopMetrics() throws IOException {
                 "          2.0\n" +
                 "        ],\n" +
                 "        \"metrics\" : {\n" +
-                "          \"test\" : 1\n" +
+                "          \"test\" : 2.0\n" +
                 "        }\n" +
                 "      }\n" +
                 "    ]\n" +
@@ -218,15 +169,10 @@ protected List<NamedXContentRegistry.Entry> getNamedXContents() {
     @Override
     protected InternalTopMetrics createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
-        return createTestInstance(name, pipelineAggregators, metaData, InternalAggregationTestCase::randomNumericDocValueFormat);
-    }
-
-    private InternalTopMetrics createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
-            Map<String, Object> metaData, Supplier<DocValueFormat> randomDocValueFormat) {
         int metricCount = between(1, 5);
         List<String> metricNames = randomMetricNames(metricCount);
         int size = between(1, 100);
-        List<InternalTopMetrics.TopMetric> topMetrics = randomTopMetrics(randomDocValueFormat, between(0, size), metricCount);
+        List<InternalTopMetrics.TopMetric> topMetrics = randomTopMetrics(between(0, size), metricCount);
         return new InternalTopMetrics(name, sortOrder, metricNames, size, topMetrics, pipelineAggregators, metaData);
     }
 
@@ -255,8 +201,7 @@ protected InternalTopMetrics mutateInstance(InternalTopMetrics instance) throws
         case 4:
             int fixedSize = size;
             int fixedMetricsSize = metricNames.size();
-            topMetrics = randomValueOtherThan(topMetrics, () -> randomTopMetrics(
-                    InternalAggregationTestCase::randomNumericDocValueFormat, between(1, fixedSize), fixedMetricsSize));
+            topMetrics = randomValueOtherThan(topMetrics, () -> randomTopMetrics(between(1, fixedSize), fixedMetricsSize));
             break;
         default:
             throw new IllegalArgumentException("bad mutation");
@@ -270,18 +215,6 @@ protected Reader<InternalTopMetrics> instanceReader() {
         return InternalTopMetrics::new;
     }
 
-    /**
-     * An extra test for parsing dates from xcontent because we can't random
-     * into {@link DocValueFormat.DateTime} because it doesn't
-     * implement {@link Object#equals(Object)}.
-     */
-    public void testFromXContentDates() throws IOException {
-        InternalTopMetrics aggregation = createTestInstance(
-                randomAlphaOfLength(3), emptyList(), emptyMap(), InternalTopMetricsTests::strictDateTime);
-        ParsedAggregation parsedAggregation = parseAndAssert(aggregation, randomBoolean(), randomBoolean());
-        assertFromXContent(aggregation, parsedAggregation);
-    }
-
     @Override
     protected void assertFromXContent(InternalTopMetrics aggregation, ParsedAggregation parsedAggregation) throws IOException {
         ParsedTopMetrics parsed = (ParsedTopMetrics) parsedAggregation;
@@ -295,14 +228,8 @@ protected void assertFromXContent(InternalTopMetrics aggregation, ParsedAggregat
             assertThat(parsedTop.getSort(), equalTo(singletonList(expectedSort)));
             assertThat(parsedTop.getMetrics().keySet(), hasSize(aggregation.getMetricNames().size()));
             for (int m = 0; m < aggregation.getMetricNames().size(); m++) {
-                String name = aggregation.getMetricNames().get(m);
-                InternalTopMetrics.MetricValue value = internalTop.getMetricValues().get(m);
-                assertThat(parsedTop.getMetrics(), hasKey(name));
-                if (value.getFormat() == DocValueFormat.RAW) {
-                    assertThat(parsedTop.getMetrics().get(name), equalTo(value.numberValue()));
-                } else {
-                    assertThat(parsedTop.getMetrics().get(name), equalTo(value.getValue().format(value.getFormat())));
-                }
+                assertThat(parsedTop.getMetrics(),
+                        hasEntry(aggregation.getMetricNames().get(m), internalTop.getMetricValues()[m]));
             }
         }
     }
@@ -322,11 +249,10 @@ protected void assertReduced(InternalTopMetrics reduced, List<InternalTopMetrics
         assertThat(reduced.getTopMetrics(), equalTo(winners));
     }
 
-    private List<InternalTopMetrics.TopMetric> randomTopMetrics(
-            Supplier<DocValueFormat> randomDocValueFormat, int length, int metricCount) {
+    private List<InternalTopMetrics.TopMetric> randomTopMetrics(int length, int metricCount) {
         return IntStream.range(0, length)
                 .mapToObj(i -> new InternalTopMetrics.TopMetric(
-                        randomDocValueFormat.get(), randomSortValue(), randomMetricValues(randomDocValueFormat, metricCount)
+                        randomNumericDocValueFormat(), randomSortValue(), randomMetricValues(metricCount)
                 ))
                 .sorted((lhs, rhs) -> sortOrder.reverseMul() * lhs.getSortValue().compareTo(rhs.getSortValue()))
                 .collect(toList());
@@ -340,18 +266,10 @@ static List<String> randomMetricNames(int metricCount) {
         return new ArrayList<>(names);
     }
 
-    private List<InternalTopMetrics.MetricValue> randomMetricValues(Supplier<DocValueFormat> randomDocValueFormat, int metricCount) {
-        return IntStream.range(0, metricCount)
-                .mapToObj(i -> new InternalTopMetrics.MetricValue(randomDocValueFormat.get(), randomSortValue()))
-                .collect(toList());
+    private double[] randomMetricValues(int metricCount) {
+        return IntStream.range(0, metricCount).mapToDouble(i -> randomDouble()).toArray();
     }
 
-    private static DocValueFormat strictDateTime() {
-        return new DocValueFormat.DateTime(
-            DateFormatter.forPattern("strict_date_time"), ZoneId.of("UTC"), DateFieldMapper.Resolution.MILLISECONDS);
-    }
-
-
     private static SortValue randomSortValue() {
         if (randomBoolean()) {
             return SortValue.from(randomLong());
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java
deleted file mode 100644
index f9e2e34140f5d..0000000000000
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.analytics.topmetrics;
-
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.elasticsearch.common.CheckedBiConsumer;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.sort.SortValue;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.analytics.topmetrics.InternalTopMetrics.MetricValue;
-import org.elasticsearch.xpack.analytics.topmetrics.InternalTopMetrics.TopMetric;
-import org.elasticsearch.xpack.analytics.topmetrics.TopMetricsAggregator.MetricSource;
-import org.elasticsearch.xpack.analytics.topmetrics.TopMetricsAggregator.Metrics;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.IntStream;
-
-import static java.util.Collections.singletonList;
-import static java.util.stream.Collectors.toList;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.notANumber;
-import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class TopMetricsAggregatorMetricsTests extends ESTestCase {
-    public void testUnmapped() throws IOException {
-        withMetric(null, (m, source) -> {
-            // Load from doc is a noop
-            m.loader(null).loadFromDoc(0, randomInt());
-            assertNullMetric(m, source, randomInt());
-        });
-    }
-
-    public void testEmptyLong() throws IOException {
-        SortedNumericDocValues values = mock(SortedNumericDocValues.class);
-        when(values.advanceExact(0)).thenReturn(false);
-        withMetric(valuesSource(values), (m, source) -> {
-            m.loader(null).loadFromDoc(0, 0);
-            assertNullMetric(m, source, 0);
-        });
-    }
-
-    public void testEmptyDouble() throws IOException {
-        SortedNumericDoubleValues values = mock(SortedNumericDoubleValues.class);
-        when(values.advanceExact(0)).thenReturn(false);
-        withMetric(valuesSource(values), (m, source) -> {
-            m.loader(null).loadFromDoc(0, 0);
-            assertNullMetric(m, source, 0);
-        });
-    }
-
-    public void testLoadLong() throws IOException {
-        long value = randomLong();
-        SortedNumericDocValues values = mock(SortedNumericDocValues.class);
-        when(values.advanceExact(0)).thenReturn(true);
-        when(values.docValueCount()).thenReturn(1);
-        when(values.nextValue()).thenReturn(value);
-        withMetric(valuesSource(values), (m, source) -> {
-            m.loader(null).loadFromDoc(0, 0);
-            assertMetricValue(m, 0, source, SortValue.from(value));
-        });
-    }
-
-    public void testLoadDouble() throws IOException {
-        double value = randomDouble();
-        SortedNumericDoubleValues values = mock(SortedNumericDoubleValues.class);
-        when(values.advanceExact(0)).thenReturn(true);
-        when(values.docValueCount()).thenReturn(1);
-        when(values.nextValue()).thenReturn(value);
-        withMetric(valuesSource(values), (m, source) -> {
-            m.loader(null).loadFromDoc(0, 0);
-            assertMetricValue(m, 0, source, SortValue.from(value));
-        });
-    }
-
-    public void testLoadAndSwapLong() throws IOException {
-        long firstValue = randomLong();
-        long secondValue = randomLong();
-        SortedNumericDocValues values = mock(SortedNumericDocValues.class);
-        when(values.advanceExact(0)).thenReturn(true);
-        when(values.advanceExact(1)).thenReturn(true);
-        when(values.docValueCount()).thenReturn(1);
-        when(values.nextValue()).thenReturn(firstValue, secondValue);
-        withMetric(valuesSource(values), (m, source) -> {
-            assertLoadTwoAndSwap(m, source, SortValue.from(firstValue), SortValue.from(secondValue));
-        });
-    }
-
-    public void testLoadAndSwapDouble() throws IOException {
-        double firstValue = randomDouble();
-        double secondValue = randomDouble();
-        SortedNumericDoubleValues values = mock(SortedNumericDoubleValues.class);
-        when(values.advanceExact(0)).thenReturn(true);
-        when(values.advanceExact(1)).thenReturn(true);
-        when(values.docValueCount()).thenReturn(1);
-        when(values.nextValue()).thenReturn(firstValue, secondValue);
-        withMetric(valuesSource(values), (m, source) -> {
-            assertLoadTwoAndSwap(m, source, SortValue.from(firstValue), SortValue.from(secondValue));
-        });
-    }
-
-    public void testManyValues() throws IOException {
-        long[] values = IntStream.range(0, between(2, 100)).mapToLong(i -> randomLong()).toArray();
-        List<ValuesSource.Numeric> valuesSources = Arrays.stream(values)
-                .mapToObj(v -> {
-                    try {
-                        SortedNumericDocValues docValues = mock(SortedNumericDocValues.class);
-                        when(docValues.advanceExact(0)).thenReturn(true);
-                        when(docValues.docValueCount()).thenReturn(1);
-                        when(docValues.nextValue()).thenReturn(v);
-                        return valuesSource(docValues);
-                    } catch (IOException e) {
-                        throw new AssertionError(e);
-                    }
-                })
-                .collect(toList());
-        withMetrics(valuesSources, (m, sources) -> {
-            m.loader(null).loadFromDoc(0, 0);
-            TopMetric metric = m.resultBuilder(DocValueFormat.RAW).build(0, SortValue.from(1));
-            assertThat(metric.getMetricValues(), hasSize(values.length));
-            for (int i = 0; i < values.length; i++) {
-                MetricSource source = sources.get(i);
-                assertThat(m.metric(source.getName(), 0), equalTo((double) values[i]));
-                assertThat(metric.getMetricValues(),
-                        hasItem(new MetricValue(source.getFormat(), SortValue.from(values[i]))));
-            }
-        });
-    }
-
-    private ValuesSource.Numeric valuesSource(SortedNumericDocValues values) throws IOException {
-        ValuesSource.Numeric source = mock(ValuesSource.Numeric.class);
-        when(source.isFloatingPoint()).thenReturn(false);
-        when(source.longValues(null)).thenReturn(values);
-        return source;
-    }
-
-    private ValuesSource.Numeric valuesSource(SortedNumericDoubleValues values) throws IOException {
-        ValuesSource.Numeric source = mock(ValuesSource.Numeric.class);
-        when(source.isFloatingPoint()).thenReturn(true);
-        when(source.doubleValues(null)).thenReturn(values);
-        return source;
-    }
-
-    private void withMetric(ValuesSource.Numeric valuesSource,
-            CheckedBiConsumer<Metrics, MetricSource, IOException> consumer) throws IOException {
-        withMetrics(singletonList(valuesSource), (m, sources) -> consumer.accept(m, sources.get(0)));
-    }
-
-    private void withMetrics(List<ValuesSource.Numeric> valuesSources,
-            CheckedBiConsumer<Metrics, List<MetricSource>, IOException> consumer) throws IOException {
-        Set<String> names = new HashSet<>();
-        List<MetricSource> sources = new ArrayList<>(valuesSources.size());
-        for (ValuesSource.Numeric valuesSource : valuesSources) {
-            String name = randomValueOtherThanMany(names::contains, () -> randomAlphaOfLength(5));
-            names.add(name);
-            sources.add(new MetricSource(name, randomDocValueFormat(), valuesSource));
-        }
-        try (Metrics m = new Metrics(1, BigArrays.NON_RECYCLING_INSTANCE, sources)) {
-            consumer.accept(m, sources);
-        }
-    }
-
-    private void assertNullMetric(Metrics m, MetricSource source, long index) {
-        DocValueFormat sortFormat = randomDocValueFormat();
-        assertThat(m.metric(source.getName(), index), notANumber());
-        TopMetric metric = m.resultBuilder(sortFormat).build(index, SortValue.from(1));
-        assertThat(metric.getSortFormat(), sameInstance(sortFormat));
-        assertThat(metric.getMetricValues(), equalTo(singletonList(null)));
-    }
-
-    private void assertMetricValue(Metrics m, long index, MetricSource source, SortValue value) {
-        DocValueFormat sortFormat = randomDocValueFormat();
-        assertThat(m.metric(source.getName(), index), equalTo(value.numberValue().doubleValue()));
-        TopMetric metric = m.resultBuilder(sortFormat).build(index, SortValue.from(1));
-        assertThat(metric.getSortValue(), equalTo(SortValue.from(1)));
-        assertThat(metric.getSortFormat(), sameInstance(sortFormat));
-        assertThat(metric.getMetricValues(), equalTo(singletonList(new MetricValue(source.getFormat(), value))));
-    }
-
-    private void assertLoadTwoAndSwap(Metrics m, MetricSource source, SortValue firstValue, SortValue secondValue) throws IOException {
-        m.loader(null).loadFromDoc(0, 0);
-        m.loader(null).loadFromDoc(1, 1);
-        assertMetricValue(m, 0, source, firstValue);
-        assertMetricValue(m, 1, source, secondValue);
-        m.swap(0, 1);
-        assertMetricValue(m, 0, source, secondValue);
-        assertMetricValue(m, 1, source, firstValue);
-        m.loader(null).loadFromDoc(2, 2); // 2 is empty
-        assertNullMetric(m, source, 2);
-        m.swap(0, 2);
-        assertNullMetric(m, source, 0);
-        assertMetricValue(m, 2, source, secondValue);
-    }
-
-    private DocValueFormat randomDocValueFormat() {
-        return randomFrom(DocValueFormat.RAW, DocValueFormat.BINARY, DocValueFormat.BOOLEAN);
-    }
-}
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
index 39dc9bf5e9fda..75f91adf569f9 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
@@ -78,6 +78,7 @@
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notANumber;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -96,12 +97,10 @@ public void testUnmappedMetric() throws IOException {
                 },
                 numberFieldType(NumberType.DOUBLE, "s"));
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
-        assertThat(result.getTopMetrics(), hasSize(1));
-        assertThat(result.getTopMetrics().get(0).getSortValue(), equalTo(SortValue.from(1.0)));
-        assertThat(result.getTopMetrics().get(0).getMetricValues(), equalTo(singletonList(null)));
+        assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.0, Double.NaN))));
     }
 
-    public void testMissingValueForDoubleMetric() throws IOException {
+    public void testMissingValueForMetric() throws IOException {
         InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(singletonList(doubleField("s", 1.0)));
                 },
@@ -109,21 +108,11 @@ public void testMissingValueForDoubleMetric() throws IOException {
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
         assertThat(result.getTopMetrics(), hasSize(1));
         assertThat(result.getTopMetrics().get(0).getSortValue(), equalTo(SortValue.from(1.0)));
-        assertThat(result.getTopMetrics().get(0).getMetricValues(), equalTo(singletonList(null)));
+        assertThat(result.getTopMetrics().get(0).getMetricValues().length, equalTo(1));
+        assertThat(result.getTopMetrics().get(0).getMetricValues()[0], notANumber());
     }
 
-    public void testMissingValueForLongMetric() throws IOException {
-        InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
-                    writer.addDocument(singletonList(longField("s", 1)));
-                },
-                longFields());
-        assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
-        assertThat(result.getTopMetrics(), hasSize(1));
-        assertThat(result.getTopMetrics().get(0).getSortValue(), equalTo(SortValue.from(1)));
-        assertThat(result.getTopMetrics().get(0).getMetricValues(), equalTo(singletonList(null)));
-    }
-
-    public void testActualValueForDoubleMetric() throws IOException {
+    public void testActualValueForMetric() throws IOException {
         InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(Arrays.asList(doubleField("s", 1.0), doubleField("m", 2.0)));
                 },
@@ -132,15 +121,6 @@ public void testActualValueForDoubleMetric() throws IOException {
         assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.0, 2.0))));
     }
 
-    public void testActualValueForLongMetric() throws IOException {
-        InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
-                    writer.addDocument(Arrays.asList(longField("s", 1), longField("m", 2)));
-                },
-                longFields());
-        assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
-        assertThat(result.getTopMetrics(), equalTo(singletonList(top(1, 2))));
-    }
-
     private InternalTopMetrics collectFromDoubles(TopMetricsAggregationBuilder builder) throws IOException {
         return collect(builder, new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(Arrays.asList(doubleField("s", 1.0), doubleField("m", 2.0)));
@@ -152,7 +132,8 @@ private InternalTopMetrics collectFromDoubles(TopMetricsAggregationBuilder build
     public void testSortByDoubleAscending() throws IOException {
         InternalTopMetrics result = collectFromDoubles(simpleBuilder(new FieldSortBuilder("s").order(SortOrder.ASC)));
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
-        assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.0, 2.0))));
+        assertThat(result.getTopMetrics(), equalTo(singletonList(
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {2.0}))));
     }
 
     public void testSortByDoubleDescending() throws IOException {
@@ -406,14 +387,13 @@ public void testManyMetrics() throws IOException {
                 ));
         InternalTopMetrics result = collect(builder, new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(Arrays.asList(doubleField("s", 1.0),
-                            doubleField("m1", 12.0), longField("m2", 22), doubleField("m3", 32.0)));
+                            doubleField("m1", 12.0), doubleField("m2", 22.0), doubleField("m3", 32.0)));
                     writer.addDocument(Arrays.asList(doubleField("s", 2.0),
-                            doubleField("m1", 13.0), longField("m2", 23), doubleField("m3", 33.0)));
+                            doubleField("m1", 13.0), doubleField("m2", 23.0), doubleField("m3", 33.0)));
                 }, manyMetricsFields());
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
         assertThat(result.getTopMetrics(), equalTo(singletonList(
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), metricValues(
-                        SortValue.from(12.0), SortValue.from(22), SortValue.from(32.0))))));
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {12.0, 22.0, 32.0}))));
     }
 
     private TopMetricsAggregationBuilder simpleBuilder() {
@@ -446,15 +426,11 @@ private MappedFieldType[] doubleFields() {
         return new MappedFieldType[] {numberFieldType(NumberType.DOUBLE, "s"), numberFieldType(NumberType.DOUBLE, "m")};
     }
 
-    private MappedFieldType[] longFields() {
-        return new MappedFieldType[] {numberFieldType(NumberType.LONG, "s"), numberFieldType(NumberType.LONG, "m")};
-    }
-
     private MappedFieldType[] manyMetricsFields() {
         return new MappedFieldType[] {
                 numberFieldType(NumberType.DOUBLE, "s"),
                 numberFieldType(NumberType.DOUBLE, "m1"),
-                numberFieldType(NumberType.LONG, "m2"),
+                numberFieldType(NumberType.DOUBLE, "m2"),
                 numberFieldType(NumberType.DOUBLE, "m3"),
         };
     }
@@ -540,29 +516,11 @@ private InternalAggregation collect(AggregationBuilder builder, Query query,
     }
 
     private InternalTopMetrics.TopMetric top(long sortValue, double... metricValues) {
-        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues(metricValues));
-    }
-
-    private InternalTopMetrics.TopMetric top(long sortValue, long... metricValues) {
-        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues(metricValues));
+        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues);
     }
 
     private InternalTopMetrics.TopMetric top(double sortValue, double... metricValues) {
-        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues(metricValues));
-    }
-
-    private List<InternalTopMetrics.MetricValue> metricValues(double... metricValues) {
-        return metricValues(Arrays.stream(metricValues).mapToObj(SortValue::from).toArray(SortValue[]::new));
-    }
-
-    private List<InternalTopMetrics.MetricValue> metricValues(long... metricValues) {
-        return metricValues(Arrays.stream(metricValues).mapToObj(SortValue::from).toArray(SortValue[]::new));
-    }
-
-    private List<InternalTopMetrics.MetricValue> metricValues(SortValue... metricValues) {
-        return Arrays.stream(metricValues)
-                .map(v -> new InternalTopMetrics.MetricValue(DocValueFormat.RAW, v))
-                .collect(toList());
+        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues);
     }
 
     /**
diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle
deleted file mode 100644
index 8aac2b9f885ac..0000000000000
--- a/x-pack/plugin/async-search/build.gradle
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-evaluationDependsOn(xpackModule('core'))
-
-apply plugin: 'elasticsearch.esplugin'
-esplugin {
-    name 'x-pack-async-search'
-    description 'A module which allows to track the progress of a search asynchronously.'
-    classname 'org.elasticsearch.xpack.search.AsyncSearch'
-    extendedPlugins = ['x-pack-core']
-}
-archivesBaseName = 'x-pack-async-search'
-
-compileJava.options.compilerArgs << "-Xlint:-rawtypes"
-compileTestJava.options.compilerArgs << "-Xlint:-rawtypes"
-
-integTest.enabled = false
-
-// add all sub-projects of the qa sub-project
-gradle.projectsEvaluated {
-  project.subprojects
-    .find { it.path == project.path + ":qa" }
-    .subprojects
-    .findAll { it.path.startsWith(project.path + ":qa") }
-    .each { check.dependsOn it.check }
-}
-
-dependencies {
-  compileOnly project(":server")
-
-  compileOnly project(path: xpackModule('core'), configuration: 'default')
-  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-  testCompile project(path: xpackModule('ilm'))
-}
-
-dependencyLicenses {
-  ignoreSha 'x-pack-core'
-}
diff --git a/x-pack/plugin/async-search/qa/build.gradle b/x-pack/plugin/async-search/qa/build.gradle
deleted file mode 100644
index 79ff4091f6d2d..0000000000000
--- a/x-pack/plugin/async-search/qa/build.gradle
+++ /dev/null
@@ -1,8 +0,0 @@
-import org.elasticsearch.gradle.test.RestIntegTestTask
-
-apply plugin: 'elasticsearch.build'
-test.enabled = false
-
-dependencies {
-  compile project(':test:framework')
-}
diff --git a/x-pack/plugin/async-search/qa/security/build.gradle b/x-pack/plugin/async-search/qa/security/build.gradle
deleted file mode 100644
index 0337749a9eebd..0000000000000
--- a/x-pack/plugin/async-search/qa/security/build.gradle
+++ /dev/null
@@ -1,19 +0,0 @@
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
-
-dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-  testCompile project(path: xpackModule('async-search'), configuration: 'runtime')
-  testCompile project(':x-pack:plugin:async-search:qa')
-}
-
-testClusters.integTest {
-  testDistribution = 'DEFAULT'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'xpack.security.enabled', 'true'
-  extraConfigFile 'roles.yml', file('roles.yml')
-  user username: "test-admin", password: 'x-pack-test-password', role: "test-admin"
-  user username: "user1", password: 'x-pack-test-password', role: "user1"
-  user username: "user2", password: 'x-pack-test-password', role: "user2"
-}
diff --git a/x-pack/plugin/async-search/qa/security/roles.yml b/x-pack/plugin/async-search/qa/security/roles.yml
deleted file mode 100644
index 4ab3be5ff0571..0000000000000
--- a/x-pack/plugin/async-search/qa/security/roles.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-# All cluster rights
-# All operations on all indices
-# Run as all users
-test-admin:
-  cluster:
-    - all
-  indices:
-    - names: '*'
-      privileges: [ all ]
-  run_as:
-    - '*'
-
-user1:
-  cluster:
-    - cluster:monitor/main
-  indices:
-    - names: ['index-user1', 'index' ]
-      privileges:
-        - read
-        - write
-        - create_index
-        - indices:admin/refresh
-
-user2:
-  cluster:
-    - cluster:monitor/main
-  indices:
-    - names: [ 'index-user2', 'index' ]
-      privileges:
-        - read
-        - write
-        - create_index
-        - indices:admin/refresh
diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
deleted file mode 100644
index 05e0132deb7d3..0000000000000
--- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.search;
-
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER;
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.elasticsearch.xpack.search.AsyncSearchIndexService.INDEX;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class AsyncSearchSecurityIT extends ESRestTestCase {
-    /**
-     * All tests run as a superuser but use <code>es-security-runas-user</code> to become a less privileged user.
-     */
-    @Override
-    protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("test-admin", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    @Before
-    public void indexDocuments() throws IOException {
-        createIndex("index", Settings.EMPTY);
-        index("index", "0", "foo", "bar");
-        refresh("index");
-
-        createIndex("index-user1", Settings.EMPTY);
-        index("index-user1", "0", "foo", "bar");
-        refresh("index-user1");
-
-        createIndex("index-user2", Settings.EMPTY);
-        index("index-user2", "0", "foo", "bar");
-        refresh("index-user2");
-    }
-
-    public void testWithUsers() throws Exception {
-        testCase("user1", "user2");
-        testCase("user2", "user1");
-    }
-
-    private void testCase(String user, String other) throws Exception {
-       for (String indexName : new String[] {"index", "index-" + user}) {
-            Response submitResp = submitAsyncSearch(indexName, "foo:bar", TimeValue.timeValueSeconds(10), user);
-            assertOK(submitResp);
-            String id = extractResponseId(submitResp);
-            Response getResp = getAsyncSearch(id, user);
-            assertOK(getResp);
-
-            // other cannot access the result
-            ResponseException exc = expectThrows(ResponseException.class, () -> getAsyncSearch(id, other));
-            assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404));
-
-            // other cannot delete the result
-            exc = expectThrows(ResponseException.class, () -> deleteAsyncSearch(id, other));
-            assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404));
-
-            // other and user cannot access the result from direct get calls
-           AsyncSearchId searchId = AsyncSearchId.decode(id);
-           for (String runAs : new String[] {user, other}) {
-               exc = expectThrows(ResponseException.class, () -> get(INDEX, searchId.getDocId(), runAs));
-               assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403));
-               assertThat(exc.getMessage(), containsString("unauthorized"));
-           }
-
-            Response delResp = deleteAsyncSearch(id, user);
-            assertOK(delResp);
-        }
-        ResponseException exc = expectThrows(ResponseException.class,
-            () -> submitAsyncSearch("index-" + other, "*", TimeValue.timeValueSeconds(10), user));
-        assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403));
-        assertThat(exc.getMessage(), containsString("unauthorized"));
-    }
-
-    static String extractResponseId(Response response) throws IOException {
-        Map<String, Object> map = toMap(response);
-        return (String) map.get("id");
-    }
-
-    static void index(String index, String id, Object... fields) throws IOException {
-        XContentBuilder document = jsonBuilder().startObject();
-        for (int i = 0; i < fields.length; i += 2) {
-            document.field((String) fields[i], fields[i + 1]);
-        }
-        document.endObject();
-        final Request request = new Request("POST", "/" + index + "/_doc/" + id);
-        request.setJsonEntity(Strings.toString(document));
-        assertOK(client().performRequest(request));
-    }
-
-    static void refresh(String index) throws IOException {
-        assertOK(adminClient().performRequest(new Request("POST", "/" + index + "/_refresh")));
-    }
-
-    static Response get(String index, String id, String user) throws IOException {
-        final Request request = new Request("GET", "/" + index + "/_doc/" + id);
-        setRunAsHeader(request, user);
-        return client().performRequest(request);
-    }
-
-    static Response submitAsyncSearch(String indexName, String query, TimeValue waitForCompletion, String user) throws IOException {
-        final Request request = new Request("POST", indexName + "/_async_search");
-        setRunAsHeader(request, user);
-        request.addParameter("q", query);
-        request.addParameter("wait_for_completion", waitForCompletion.toString());
-        // we do the cleanup explicitly
-        request.addParameter("clean_on_completion", "false");
-        return client().performRequest(request);
-    }
-
-    static Response getAsyncSearch(String id, String user) throws IOException {
-        final Request request = new Request("GET",  "/_async_search/" + id);
-        setRunAsHeader(request, user);
-        request.addParameter("wait_for_completion", "0ms");
-        return client().performRequest(request);
-    }
-
-    static Response deleteAsyncSearch(String id, String user) throws IOException {
-        final Request request = new Request("DELETE",  "/_async_search/" + id);
-        setRunAsHeader(request, user);
-        return client().performRequest(request);
-    }
-
-    static Map<String, Object> toMap(Response response) throws IOException {
-        return toMap(EntityUtils.toString(response.getEntity()));
-    }
-
-    static Map<String, Object> toMap(String response) {
-        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
-    }
-
-    static void setRunAsHeader(Request request, String user) {
-        final RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
-        builder.addHeader(RUN_AS_USER_HEADER, user);
-        request.setOptions(builder);
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java
deleted file mode 100644
index 580f7d8a5560a..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.plugins.ActionPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestHandler;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.watcher.ResourceWatcherService;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.function.Supplier;
-
-public final class AsyncSearch extends Plugin implements ActionPlugin {
-    private final Settings settings;
-
-    public AsyncSearch(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return Arrays.asList(
-            new ActionHandler<>(SubmitAsyncSearchAction.INSTANCE, TransportSubmitAsyncSearchAction.class),
-            new ActionHandler<>(GetAsyncSearchAction.INSTANCE, TransportGetAsyncSearchAction.class),
-            new ActionHandler<>(DeleteAsyncSearchAction.INSTANCE, TransportDeleteAsyncSearchAction.class)
-        );
-    }
-
-    @Override
-    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
-                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
-                                             IndexNameExpressionResolver indexNameExpressionResolver,
-                                             Supplier<DiscoveryNodes> nodesInCluster) {
-        return Arrays.asList(
-            new RestSubmitAsyncSearchAction(),
-            new RestGetAsyncSearchAction(),
-            new RestDeleteAsyncSearchAction()
-        );
-    }
-
-    @Override
-    public Collection<Object> createComponents(Client client,
-                                               ClusterService clusterService,
-                                               ThreadPool threadPool,
-                                               ResourceWatcherService resourceWatcherService,
-                                               ScriptService scriptService,
-                                               NamedXContentRegistry xContentRegistry,
-                                               Environment environment,
-                                               NodeEnvironment nodeEnvironment,
-                                               NamedWriteableRegistry namedWriteableRegistry,
-                                               IndexNameExpressionResolver indexNameExpressionResolver) {
-        if (DiscoveryNode.isDataNode(environment.settings())) {
-            // only data nodes should be eligible to run the maintenance service.
-            AsyncSearchIndexService indexService =
-                new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, namedWriteableRegistry);
-            AsyncSearchMaintenanceService maintenanceService =
-                new AsyncSearchMaintenanceService(nodeEnvironment.nodeId(), threadPool, indexService, TimeValue.timeValueHours(1));
-            clusterService.addListener(maintenanceService);
-            return Collections.singletonList(maintenanceService);
-        } else {
-            return Collections.emptyList();
-        }
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java
deleted file mode 100644
index ef8b47ee00312..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.tasks.TaskId;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Base64;
-import java.util.Objects;
-
-/**
- * A class that contains all information related to a submitted async search.
- */
-class AsyncSearchId {
-    private final String docId;
-    private final TaskId taskId;
-    private final String encoded;
-
-    AsyncSearchId(String docId, TaskId taskId) {
-        this.docId = docId;
-        this.taskId = taskId;
-        this.encoded = encode(docId, taskId);
-    }
-
-    /**
-     * The document id of the response in the index if the task is not running.
-     */
-    String getDocId() {
-        return docId;
-    }
-
-    /**
-     * The {@link TaskId} of the async search in the task manager.
-     */
-    TaskId getTaskId() {
-        return taskId;
-    }
-
-    /**
-     * Gets the encoded string that represents this search.
-     */
-    String getEncoded() {
-        return encoded;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        AsyncSearchId searchId = (AsyncSearchId) o;
-        return docId.equals(searchId.docId) &&
-            taskId.equals(searchId.taskId);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(docId, taskId);
-    }
-
-    @Override
-    public String toString() {
-        return "AsyncSearchId{" +
-            "docId='" + docId + '\'' +
-            ", taskId=" + taskId +
-            '}';
-    }
-
-    /**
-     * Encodes the informations needed to retrieve a async search response
-     * in a base64 encoded string.
-     */
-    static String encode(String docId, TaskId taskId) {
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            out.writeString(docId);
-            out.writeString(taskId.toString());
-            return Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(out.bytes()));
-        } catch (IOException e) {
-            throw new IllegalArgumentException(e);
-        }
-    }
-
-    /**
-     * Decodes a base64 encoded string into an {@link AsyncSearchId} that can be used
-     * to retrieve the response of an async search.
-     */
-    static AsyncSearchId decode(String id) {
-        final AsyncSearchId searchId;
-        try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getUrlDecoder().decode(id)))) {
-            searchId = new AsyncSearchId(in.readString(), new TaskId(in.readString()));
-            if (in.available() > 0) {
-                throw new IllegalArgumentException("invalid id:[" + id + "]");
-            }
-        } catch (IOException e) {
-            throw new IllegalArgumentException("invalid id:[" + id + "]");
-        }
-        return searchId;
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
deleted file mode 100644
index d1ec9fac7c06f..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.OriginSettingClient;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskManager;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
-import org.elasticsearch.xpack.core.security.SecurityContext;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
-import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
-import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY;
-
-/**
- * A service that exposes the CRUD operations for the async-search index.
- */
-class AsyncSearchIndexService {
-    private static final Logger logger = LogManager.getLogger(AsyncSearchIndexService.class);
-
-    public static final String INDEX = ".async-search";
-
-    public static final String HEADERS_FIELD = "headers";
-    public static final String EXPIRATION_TIME_FIELD = "expiration_time";
-    public static final String RESULT_FIELD = "result";
-
-    public static Settings settings() {
-        return Settings.builder()
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-            .build();
-    }
-
-    public static XContentBuilder mappings() throws IOException {
-        XContentBuilder builder = jsonBuilder()
-            .startObject()
-                .startObject(SINGLE_MAPPING_NAME)
-                    .startObject("_meta")
-                        .field("version", Version.CURRENT)
-                    .endObject()
-                    .field("dynamic", "strict")
-                    .startObject("properties")
-                        .startObject(HEADERS_FIELD)
-                            .field("type", "object")
-                            .field("enabled", "false")
-                        .endObject()
-                        .startObject(RESULT_FIELD)
-                            .field("type", "object")
-                            .field("enabled", "false")
-                        .endObject()
-                        .startObject(EXPIRATION_TIME_FIELD)
-                            .field("type", "long")
-                        .endObject()
-                    .endObject()
-                .endObject()
-            .endObject();
-        return builder;
-    }
-
-    private final ClusterService clusterService;
-    private final Client client;
-    private final SecurityContext securityContext;
-    private final NamedWriteableRegistry registry;
-
-    AsyncSearchIndexService(ClusterService clusterService,
-                            ThreadContext threadContext,
-                            Client client,
-                            NamedWriteableRegistry registry) {
-        this.clusterService = clusterService;
-        this.securityContext = new SecurityContext(clusterService.getSettings(), threadContext);
-        this.client = new OriginSettingClient(client, ASYNC_SEARCH_ORIGIN);
-        this.registry = registry;
-    }
-
-    /**
-     * Returns the internal client with origin.
-     */
-    Client getClient() {
-        return client;
-    }
-
-    /**
-     * Creates the index with the expected settings and mappings if it doesn't exist.
-     */
-    void createIndexIfNecessary(ActionListener<Void> listener) {
-        if (clusterService.state().routingTable().hasIndex(AsyncSearchIndexService.INDEX) == false) {
-            try {
-                client.admin().indices().prepareCreate(INDEX)
-                    .setSettings(settings())
-                    .setMapping(mappings())
-                    .execute(ActionListener.wrap(
-                        resp -> listener.onResponse(null),
-                        exc -> {
-                            if (ExceptionsHelper.unwrapCause(exc) instanceof ResourceAlreadyExistsException) {
-                                listener.onResponse(null);
-                            } else {
-                                logger.error("failed to create async-search index", exc);
-                                listener.onFailure(exc);
-                            }
-                        }));
-            } catch (Exception exc) {
-                logger.error("failed to create async-search index", exc);
-                listener.onFailure(exc);
-            }
-        } else {
-            listener.onResponse(null);
-        }
-    }
-
-    /**
-     * Stores the initial response with the original headers of the authenticated user
-     * and the expected expiration time.
-     */
-    void storeInitialResponse(String docId,
-                              Map<String, String> headers,
-                              AsyncSearchResponse response,
-                              ActionListener<IndexResponse> listener) throws IOException {
-        Map<String, Object> source = new HashMap<>();
-        source.put(HEADERS_FIELD, headers);
-        source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime());
-        source.put(RESULT_FIELD, encodeResponse(response));
-        IndexRequest indexRequest = new IndexRequest(INDEX)
-            .create(true)
-            .id(docId)
-            .source(source, XContentType.JSON);
-        createIndexIfNecessary(ActionListener.wrap(v -> client.index(indexRequest, listener), listener::onFailure));
-    }
-
-    /**
-     * Stores the final response if the place-holder document is still present (update).
-     */
-    void storeFinalResponse(String docId,
-                            AsyncSearchResponse response,
-                            ActionListener<UpdateResponse> listener) throws IOException {
-        Map<String, Object> source = new HashMap<>();
-        source.put(RESULT_FIELD, encodeResponse(response));
-        UpdateRequest request = new UpdateRequest()
-            .index(INDEX)
-            .id(docId)
-            .doc(source, XContentType.JSON);
-        createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure));
-    }
-
-    /**
-     * Updates the expiration time of the provided <code>docId</code> if the place-holder
-     * document is still present (update).
-     */
-    void updateExpirationTime(String docId,
-                              long expirationTimeMillis,
-                              ActionListener<UpdateResponse> listener) {
-        Map<String, Object> source = Collections.singletonMap(EXPIRATION_TIME_FIELD, expirationTimeMillis);
-        UpdateRequest request = new UpdateRequest().index(INDEX)
-            .id(docId)
-            .doc(source, XContentType.JSON);
-        createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure));
-    }
-
-    /**
-     * Deletes the provided <code>searchId</code> from the index if present.
-     */
-    void deleteResponse(AsyncSearchId searchId,
-                        boolean failIfNotFound,
-                        ActionListener<AcknowledgedResponse> listener) {
-        DeleteRequest request = new DeleteRequest(INDEX).id(searchId.getDocId());
-        createIndexIfNecessary(
-            ActionListener.wrap(v -> client.delete(request,
-                ActionListener.wrap(
-                    resp -> {
-                        if (resp.status() == RestStatus.NOT_FOUND && failIfNotFound) {
-                            listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
-                        } else {
-                            listener.onResponse(new AcknowledgedResponse(true));
-                        }
-                    },
-                    exc -> {
-                        logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc);
-                        listener.onFailure(exc);
-                    })),
-                listener::onFailure));
-    }
-
-    /**
-     * Returns the {@link AsyncSearchTask} if the provided <code>searchId</code>
-     * is registered in the task manager, <code>null</code> otherwise.
-     *
-     * This method throws a {@link ResourceNotFoundException} if the authenticated user
-     * is not the creator of the original task.
-     */
-    AsyncSearchTask getTask(TaskManager taskManager, AsyncSearchId searchId) throws IOException {
-        Task task = taskManager.getTask(searchId.getTaskId().getId());
-        if (task instanceof AsyncSearchTask == false) {
-            return null;
-        }
-        AsyncSearchTask searchTask = (AsyncSearchTask) task;
-        if (searchTask.getSearchId().equals(searchId) == false) {
-            return null;
-        }
-
-        // Check authentication for the user
-        final Authentication auth = securityContext.getAuthentication();
-        if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) {
-            throw new ResourceNotFoundException(searchId.getEncoded() + " not found");
-        }
-        return searchTask;
-    }
-
-    /**
-     * Gets the response from the index if present, or delegate a {@link ResourceNotFoundException}
-     * failure to the provided listener if not.
-     */
-    void getResponse(AsyncSearchId searchId,
-                     ActionListener<AsyncSearchResponse> listener) {
-        final Authentication current = securityContext.getAuthentication();
-        GetRequest internalGet = new GetRequest(INDEX)
-            .preference(searchId.getEncoded())
-            .id(searchId.getDocId());
-        client.get(internalGet, ActionListener.wrap(
-            get -> {
-                if (get.isExists() == false) {
-                    listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
-                    return;
-                }
-
-                // check the authentication of the current user against the user that initiated the async search
-                @SuppressWarnings("unchecked")
-                Map<String, String> headers = (Map<String, String>) get.getSource().get(HEADERS_FIELD);
-                if (ensureAuthenticatedUserIsSame(headers, current) == false) {
-                    listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
-                    return;
-                }
-
-                String encoded = (String) get.getSource().get(RESULT_FIELD);
-                listener.onResponse(encoded != null ? decodeResponse(encoded) : null);
-            },
-            listener::onFailure
-        ));
-    }
-
-    /**
-     * Extracts the authentication from the original headers and checks that it matches
-     * the current user. This function returns always <code>true</code> if the provided
-     * <code>headers</code> do not contain any authentication.
-     */
-    boolean ensureAuthenticatedUserIsSame(Map<String, String> originHeaders, Authentication current) throws IOException {
-        if (originHeaders == null || originHeaders.containsKey(AUTHENTICATION_KEY) == false) {
-            // no authorization attached to the original request
-            return true;
-        }
-        if (current == null) {
-            // origin is an authenticated user but current is not
-            return false;
-        }
-        Authentication origin = AuthenticationContextSerializer.decode(originHeaders.get(AUTHENTICATION_KEY));
-        return ensureAuthenticatedUserIsSame(origin, current);
-    }
-
-    /**
-     * Compares the {@link Authentication} that was used to create the {@link AsyncSearchId} with the
-     * current authentication.
-     */
-    boolean ensureAuthenticatedUserIsSame(Authentication original, Authentication current) {
-        final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal());
-        final boolean sameRealmType;
-        if (original.getUser().isRunAs()) {
-            if (current.getUser().isRunAs()) {
-                sameRealmType = original.getLookedUpBy().getType().equals(current.getLookedUpBy().getType());
-            }  else {
-                sameRealmType = original.getLookedUpBy().getType().equals(current.getAuthenticatedBy().getType());
-            }
-        } else if (current.getUser().isRunAs()) {
-            sameRealmType = original.getAuthenticatedBy().getType().equals(current.getLookedUpBy().getType());
-        } else {
-            sameRealmType = original.getAuthenticatedBy().getType().equals(current.getAuthenticatedBy().getType());
-        }
-        return samePrincipal && sameRealmType;
-    }
-
-    /**
-     * Encode the provided response in a binary form using base64 encoding.
-     */
-    String encodeResponse(AsyncSearchResponse response) throws IOException {
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            Version.writeVersion(Version.CURRENT, out);
-            response.writeTo(out);
-            return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes()));
-        }
-    }
-
-    /**
-     * Decode the provided base-64 bytes into a {@link AsyncSearchResponse}.
-     */
-    AsyncSearchResponse decodeResponse(String value) throws IOException {
-        try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getDecoder().decode(value)))) {
-            try (StreamInput in = new NamedWriteableAwareStreamInput(buf, registry)) {
-                in.setVersion(Version.readVersion(in));
-                return new AsyncSearchResponse(in);
-            }
-        }
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java
deleted file mode 100644
index 1cac9c0eaf458..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.search;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.gateway.GatewayService;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.reindex.DeleteByQueryAction;
-import org.elasticsearch.index.reindex.DeleteByQueryRequest;
-import org.elasticsearch.threadpool.Scheduler;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.elasticsearch.xpack.search.AsyncSearchIndexService.EXPIRATION_TIME_FIELD;
-
-/**
- * A service that runs a periodic cleanup over the async-search index.
- */
-class AsyncSearchMaintenanceService implements Releasable, ClusterStateListener {
-    private static final Logger logger = LogManager.getLogger(AsyncSearchMaintenanceService.class);
-
-    private final String localNodeId;
-    private final ThreadPool threadPool;
-    private final AsyncSearchIndexService indexService;
-    private final TimeValue delay;
-
-    private final AtomicBoolean isCleanupRunning = new AtomicBoolean(false);
-    private final AtomicBoolean isClosed = new AtomicBoolean(false);
-    private volatile Scheduler.Cancellable cancellable;
-
-    AsyncSearchMaintenanceService(String localNodeId,
-                                  ThreadPool threadPool,
-                                  AsyncSearchIndexService indexService,
-                                  TimeValue delay) {
-        this.localNodeId = localNodeId;
-        this.threadPool = threadPool;
-        this.indexService = indexService;
-        this.delay = delay;
-    }
-
-    @Override
-    public void clusterChanged(ClusterChangedEvent event) {
-        final ClusterState state = event.state();
-        if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
-            // Wait until the gateway has recovered from disk.
-            return;
-        }
-        tryStartCleanup(state);
-    }
-
-    void tryStartCleanup(ClusterState state) {
-        if (isClosed.get()) {
-            return;
-        }
-        IndexRoutingTable indexRouting = state.routingTable().index(AsyncSearchIndexService.INDEX);
-        if (indexRouting == null) {
-            if (isCleanupRunning.compareAndSet(true, false)) {
-                close();
-            }
-            return;
-        }
-        String primaryNodeId = indexRouting.shard(0).primaryShard().currentNodeId();
-        if (localNodeId.equals(primaryNodeId)) {
-            if (isCleanupRunning.compareAndSet(false, true)) {
-                executeNextCleanup();
-            }
-        } else if (isCleanupRunning.compareAndSet(true, false)) {
-            close();
-        }
-    }
-
-    synchronized void executeNextCleanup() {
-        if (isClosed.get() == false && isCleanupRunning.get()) {
-            long nowInMillis = System.currentTimeMillis();
-            DeleteByQueryRequest toDelete = new DeleteByQueryRequest()
-                .setQuery(QueryBuilders.rangeQuery(EXPIRATION_TIME_FIELD).lte(nowInMillis));
-            indexService.getClient()
-                .execute(DeleteByQueryAction.INSTANCE, toDelete, ActionListener.wrap(() -> scheduleNextCleanup()));
-        }
-    }
-
-    synchronized void scheduleNextCleanup() {
-        if (isClosed.get() == false && isCleanupRunning.get()) {
-            try {
-                cancellable = threadPool.schedule(this::executeNextCleanup, delay, ThreadPool.Names.GENERIC);
-            } catch (EsRejectedExecutionException e) {
-                if (e.isExecutorShutdown()) {
-                    logger.debug("failed to schedule next maintenance task; shutting down", e);
-                } else {
-                    throw e;
-                }
-            }
-        }
-    }
-
-    @Override
-    public void close() {
-        if (cancellable != null && cancellable.isCancelled() == false) {
-            cancellable.cancel();
-        }
-        isClosed.compareAndSet(false, true);
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
deleted file mode 100644
index cb2862a087774..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
-import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
-import org.elasticsearch.action.search.SearchProgressActionListener;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchResponse.Clusters;
-import org.elasticsearch.action.search.SearchShard;
-import org.elasticsearch.action.search.SearchTask;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.threadpool.Scheduler.Cancellable;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-/**
- * Task that tracks the progress of a currently running {@link SearchRequest}.
- */
-final class AsyncSearchTask extends SearchTask {
-    private final AsyncSearchId searchId;
-    private final Client client;
-    private final ThreadPool threadPool;
-    private final Supplier<ReduceContext> reduceContextSupplier;
-    private final Listener progressListener;
-
-    private final Map<String, String> originHeaders;
-
-    private boolean hasInitialized;
-    private boolean hasCompleted;
-    private long completionId;
-    private final List<Runnable> initListeners = new ArrayList<>();
-    private final Map<Long, Consumer<AsyncSearchResponse>> completionListeners = new HashMap<>();
-
-    private volatile long expirationTimeMillis;
-    private final AtomicBoolean isCancelling = new AtomicBoolean(false);
-
-    private AtomicReference<MutableSearchResponse> searchResponse;
-
-    /**
-     * Creates an instance of {@link AsyncSearchTask}.
-     *
-     * @param id The id of the task.
-     * @param type The type of the task.
-     * @param action The action name.
-     * @param parentTaskId The parent task id.
-     * @param originHeaders All the request context headers.
-     * @param taskHeaders The filtered request headers for the task.
-     * @param searchId The {@link AsyncSearchId} of the task.
-     * @param threadPool The threadPool to schedule runnable.
-     * @param reduceContextSupplier A supplier to create final reduce contexts.
-     */
-    AsyncSearchTask(long id,
-                    String type,
-                    String action,
-                    TaskId parentTaskId,
-                    TimeValue keepAlive,
-                    Map<String, String> originHeaders,
-                    Map<String, String> taskHeaders,
-                    AsyncSearchId searchId,
-                    Client client,
-                    ThreadPool threadPool,
-                    Supplier<ReduceContext> reduceContextSupplier) {
-        super(id, type, action, "async_search", parentTaskId, taskHeaders);
-        this.expirationTimeMillis = getStartTime() + keepAlive.getMillis();
-        this.originHeaders = originHeaders;
-        this.searchId = searchId;
-        this.client = client;
-        this.threadPool = threadPool;
-        this.reduceContextSupplier = reduceContextSupplier;
-        this.progressListener = new Listener();
-        this.searchResponse = new AtomicReference<>();
-        setProgressListener(progressListener);
-    }
-
-    /**
-     * Returns all of the request contexts headers
-     */
-    Map<String, String> getOriginHeaders() {
-        return originHeaders;
-    }
-
-    /**
-     * Returns the {@link AsyncSearchId} of the task
-     */
-    AsyncSearchId getSearchId() {
-        return searchId;
-    }
-
-    Listener getSearchProgressActionListener() {
-        return progressListener;
-    }
-
-    /**
-     * Update the expiration time of the (partial) response.
-     */
-    public void setExpirationTime(long expirationTimeMillis) {
-        this.expirationTimeMillis = expirationTimeMillis;
-    }
-
-    /**
-     * Cancels the running task and its children.
-     */
-    public void cancelTask(Runnable runnable) {
-        if (isCancelled() == false && isCancelling.compareAndSet(false, true)) {
-            CancelTasksRequest req = new CancelTasksRequest().setTaskId(searchId.getTaskId());
-            client.admin().cluster().cancelTasks(req, new ActionListener<>() {
-                @Override
-                public void onResponse(CancelTasksResponse cancelTasksResponse) {
-                    runnable.run();
-                }
-
-                @Override
-                public void onFailure(Exception exc) {
-                    // cancelling failed
-                    isCancelling.compareAndSet(true, false);
-                    runnable.run();
-                }
-            });
-        } else {
-            runnable.run();
-       }
-    }
-
-    @Override
-    protected void onCancelled() {
-        super.onCancelled();
-        isCancelling.compareAndSet(true, false);
-    }
-
-    /**
-     * Creates a listener that listens for an {@link AsyncSearchResponse} and executes the
-     * consumer when the task is finished or when the provided <code>waitForCompletion</code>
-     * timeout occurs. In such case the consumed {@link AsyncSearchResponse} will contain partial results.
-     */
-    public void addCompletionListener(ActionListener<AsyncSearchResponse> listener, TimeValue waitForCompletion) {
-        boolean executeImmediately = false;
-        long startTime = threadPool.relativeTimeInMillis();
-        synchronized (this) {
-            if (hasCompleted) {
-                executeImmediately = true;
-            } else {
-                addInitListener(() -> {
-                    final TimeValue remainingWaitForCompletion;
-                    if (waitForCompletion.getMillis() > 0) {
-                        long elapsedTime = threadPool.relativeTimeInMillis() - startTime;
-                        // subtract the initialization time from the provided waitForCompletion.
-                        remainingWaitForCompletion = TimeValue.timeValueMillis(Math.max(0, waitForCompletion.getMillis() - elapsedTime));
-                    } else {
-                        remainingWaitForCompletion = TimeValue.ZERO;
-                    }
-                    internalAddCompletionListener(listener, remainingWaitForCompletion);
-                });
-            }
-        }
-        if (executeImmediately) {
-            listener.onResponse(getResponse());
-        }
-    }
-
-    /**
-     * Creates a listener that listens for an {@link AsyncSearchResponse} and executes the
-     * consumer when the task is finished.
-     */
-    public void addCompletionListener(Consumer<AsyncSearchResponse>  listener) {
-        boolean executeImmediately = false;
-        synchronized (this) {
-            if (hasCompleted) {
-                executeImmediately = true;
-            } else {
-                completionListeners.put(completionId++, listener::accept);
-            }
-        }
-        if (executeImmediately) {
-            listener.accept(getResponse());
-        }
-    }
-
-    private void internalAddCompletionListener(ActionListener<AsyncSearchResponse> listener, TimeValue waitForCompletion) {
-        boolean executeImmediately = false;
-        synchronized (this) {
-            if (hasCompleted || waitForCompletion.getMillis() == 0) {
-                executeImmediately = true;
-            } else {
-                // ensure that we consumes the listener only once
-                AtomicBoolean hasRun = new AtomicBoolean(false);
-                long id = completionId++;
-
-                final Cancellable cancellable;
-                try {
-                    cancellable = threadPool.schedule(() -> {
-                        if (hasRun.compareAndSet(false, true)) {
-                            // timeout occurred before completion
-                            removeCompletionListener(id);
-                            listener.onResponse(getResponse());
-                        }
-                    }, waitForCompletion, "generic");
-                } catch (EsRejectedExecutionException exc) {
-                    listener.onFailure(exc);
-                    return;
-                }
-                completionListeners.put(id, resp -> {
-                    if (hasRun.compareAndSet(false, true)) {
-                        // completion occurred before timeout
-                        cancellable.cancel();
-                        listener.onResponse(resp);
-                    }
-                });
-            }
-        }
-        if (executeImmediately) {
-            listener.onResponse(getResponse());
-        }
-    }
-
-    private void removeCompletionListener(long id) {
-        synchronized (this) {
-            if (hasCompleted == false) {
-                completionListeners.remove(id);
-            }
-        }
-    }
-
-    private void addInitListener(Runnable listener) {
-        boolean executeImmediately = false;
-        synchronized (this) {
-            if (hasInitialized) {
-                executeImmediately = true;
-            } else {
-                initListeners.add(listener);
-            }
-        }
-        if (executeImmediately) {
-            listener.run();
-        }
-    }
-
-    private void executeInitListeners() {
-        synchronized (this) {
-            if (hasInitialized) {
-                return;
-            }
-            hasInitialized = true;
-        }
-        for (Runnable listener : initListeners) {
-            listener.run();
-        }
-        initListeners.clear();
-    }
-
-    private void executeCompletionListeners() {
-        synchronized (this) {
-            if (hasCompleted) {
-                return;
-            }
-            hasCompleted = true;
-        }
-        AsyncSearchResponse finalResponse = getResponse();
-        for (Consumer<AsyncSearchResponse> listener : completionListeners.values()) {
-            listener.accept(finalResponse);
-        }
-        completionListeners.clear();
-    }
-
-    private AsyncSearchResponse getResponse() {
-        assert searchResponse.get() != null;
-        return searchResponse.get().toAsyncSearchResponse(this, expirationTimeMillis);
-    }
-
-    // cancels the task if it expired
-    private void checkExpiration() {
-        long now = System.currentTimeMillis();
-        if (expirationTimeMillis < now) {
-            cancelTask(() -> {});
-        }
-    }
-
-    class Listener extends SearchProgressActionListener {
-        @Override
-        protected void onQueryResult(int shardIndex) {
-            checkExpiration();
-        }
-
-        @Override
-        protected void onFetchResult(int shardIndex) {
-            checkExpiration();
-        }
-
-        @Override
-        protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
-            // best effort to cancel expired tasks
-            checkExpiration();
-            searchResponse.get().addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget));
-        }
-
-        @Override
-        protected void onFetchFailure(int shardIndex, Exception exc) {
-            checkExpiration();
-        }
-
-        @Override
-        protected void onListShards(List<SearchShard> shards, List<SearchShard> skipped, Clusters clusters, boolean fetchPhase) {
-            // best effort to cancel expired tasks
-            checkExpiration();
-            searchResponse.compareAndSet(null,
-                new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier));
-            executeInitListeners();
-        }
-
-        @Override
-        public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
-            // best effort to cancel expired tasks
-            checkExpiration();
-            searchResponse.get().updatePartialResponse(shards.size(),
-                new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
-                    null, null, false, null, reducePhase), aggs == null);
-        }
-
-        @Override
-        public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
-            // best effort to cancel expired tasks
-            checkExpiration();
-            searchResponse.get().updatePartialResponse(shards.size(),
-                new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
-                    null, null, false, null, reducePhase), true);
-        }
-
-        @Override
-        public void onResponse(SearchResponse response) {
-            searchResponse.get().updateFinalResponse(response.getSuccessfulShards(), response.getInternalResponse());
-            executeCompletionListeners();
-        }
-
-        @Override
-        public void onFailure(Exception exc) {
-            if (searchResponse.get() == null) {
-                // if the failure occurred before calling onListShards
-                searchResponse.compareAndSet(null,
-                    new MutableSearchResponse(-1, -1, null, reduceContextSupplier));
-            }
-            searchResponse.get().updateWithFailure(exc);
-            executeInitListeners();
-            executeCompletionListeners();
-        }
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
deleted file mode 100644
index 1949aceb6bd91..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchResponse.Clusters;
-import org.elasticsearch.action.search.SearchResponseSections;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Supplier;
-
-import static java.util.Collections.singletonList;
-import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
-import static org.elasticsearch.search.aggregations.InternalAggregations.topLevelReduce;
-
-/**
- * A mutable search response that allows to update and create partial response synchronously.
- * Synchronized methods ensure that updates of the content are blocked if another thread is
- * creating an async response concurrently. This limits the number of final reduction that can
- * run concurrently to 1 and ensures that we pause the search progress when an {@link AsyncSearchResponse} is built.
- */
-class MutableSearchResponse {
-    private final int totalShards;
-    private final int skippedShards;
-    private final Clusters clusters;
-    private final AtomicArray<ShardSearchFailure> shardFailures;
-    private final Supplier<ReduceContext> reduceContextSupplier;
-
-    private int version;
-    private boolean isPartial;
-    private boolean isFinalReduce;
-    private int successfulShards;
-    private SearchResponseSections sections;
-    private ElasticsearchException failure;
-
-    private boolean frozen;
-
-    /**
-     * Creates a new mutable search response.
-     *
-     * @param totalShards The number of shards that participate in the request, or -1 to indicate a failure.
-     * @param skippedShards The number of skipped shards, or -1 to indicate a failure.
-     * @param clusters The remote clusters statistics.
-     * @param reduceContextSupplier A supplier to run final reduce on partial aggregations.
-     */
-    MutableSearchResponse(int totalShards, int skippedShards, Clusters clusters, Supplier<ReduceContext> reduceContextSupplier) {
-        this.totalShards = totalShards;
-        this.skippedShards = skippedShards;
-        this.clusters = clusters;
-        this.reduceContextSupplier = reduceContextSupplier;
-        this.version = 0;
-        this.shardFailures = totalShards == -1 ? null : new AtomicArray<>(totalShards-skippedShards);
-        this.isPartial = true;
-        this.sections = totalShards == -1 ? null : new InternalSearchResponse(
-            new SearchHits(SearchHits.EMPTY, new TotalHits(0, GREATER_THAN_OR_EQUAL_TO), Float.NaN),
-            null, null, null, false, null, 0);
-    }
-
-    /**
-     * Updates the response with the partial {@link SearchResponseSections} merged from #<code>successfulShards</code>
-     * shards.
-     */
-    synchronized void updatePartialResponse(int successfulShards, SearchResponseSections newSections, boolean isFinalReduce) {
-        failIfFrozen();
-        if (newSections.getNumReducePhases() < sections.getNumReducePhases()) {
-            // should never happen since partial response are updated under a lock
-            // in the search phase controller
-            throw new IllegalStateException("received partial response out of order: "
-                + newSections.getNumReducePhases() + " < " + sections.getNumReducePhases());
-        }
-        ++ version;
-        this.successfulShards = successfulShards;
-        this.sections = newSections;
-        this.isPartial = true;
-        this.isFinalReduce = isFinalReduce;
-    }
-
-    /**
-     * Updates the response with the final {@link SearchResponseSections} merged from #<code>successfulShards</code>
-     * shards.
-     */
-    synchronized void updateFinalResponse(int successfulShards, SearchResponseSections newSections) {
-        failIfFrozen();
-        ++ version;
-        this.successfulShards = successfulShards;
-        this.sections = newSections;
-        this.isPartial = false;
-        this.isFinalReduce = true;
-        this.frozen = true;
-    }
-
-    /**
-     * Updates the response with a fatal failure. This method preserves the partial response
-     * received from previous updates
-     */
-    synchronized void updateWithFailure(Exception exc) {
-        failIfFrozen();
-        ++ version;
-        this.isPartial = true;
-        this.failure = ElasticsearchException.guessRootCauses(exc)[0];
-        this.frozen = true;
-    }
-
-    /**
-     * Adds a shard failure concurrently (non-blocking).
-     */
-    void addShardFailure(int shardIndex, ShardSearchFailure failure) {
-        synchronized (this) {
-            failIfFrozen();
-        }
-        shardFailures.set(shardIndex, failure);
-    }
-
-    /**
-     * Creates an {@link AsyncSearchResponse} based on the current state of the mutable response.
-     * The final reduce of the aggregations is executed if needed (partial response).
-     * This method is synchronized to ensure that we don't perform final reduces concurrently.
-     */
-    synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, long expirationTime) {
-        final SearchResponse resp;
-        if (totalShards != -1) {
-            if (sections.aggregations() != null && isFinalReduce == false) {
-                InternalAggregations oldAggs = (InternalAggregations) sections.aggregations();
-                InternalAggregations newAggs = topLevelReduce(singletonList(oldAggs), reduceContextSupplier.get());
-                sections = new InternalSearchResponse(sections.hits(), newAggs, sections.suggest(),
-                    null, sections.timedOut(), sections.terminatedEarly(), sections.getNumReducePhases());
-                isFinalReduce = true;
-            }
-            long tookInMillis = TimeValue.timeValueNanos(System.nanoTime() - task.getStartTimeNanos()).getMillis();
-            resp = new SearchResponse(sections, null, totalShards, successfulShards,
-                skippedShards, tookInMillis, buildShardFailures(), clusters);
-        } else {
-            resp = null;
-        }
-        return new AsyncSearchResponse(task.getSearchId().getEncoded(), version, resp, failure, isPartial,
-            frozen == false, task.getStartTime(), expirationTime);
-    }
-
-    private void failIfFrozen() {
-        if (frozen) {
-            throw new IllegalStateException("invalid update received after the completion of the request");
-        }
-    }
-
-    private ShardSearchFailure[] buildShardFailures() {
-        if (shardFailures == null) {
-            return new ShardSearchFailure[0];
-        }
-        List<ShardSearchFailure> failures = new ArrayList<>();
-        for (int i = 0; i < shardFailures.length(); i++) {
-            ShardSearchFailure failure = shardFailures.get(i);
-            if (failure != null) {
-                failures.add(failure);
-            }
-        }
-        return failures.toArray(ShardSearchFailure[]::new);
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java
deleted file mode 100644
index faab51dc3af73..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestHandler.Route;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-
-
-import java.io.IOException;
-import java.util.List;
-
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
-import static org.elasticsearch.rest.RestRequest.Method.DELETE;
-
-public class RestDeleteAsyncSearchAction extends BaseRestHandler  {
-    @Override
-    public List<Route> routes() {
-        return unmodifiableList(asList(
-            new Route(DELETE, "/_async_search/{id}")));
-    }
-
-    @Override
-    public String getName() {
-        return "async_search_delete_action";
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        DeleteAsyncSearchAction.Request delete = new DeleteAsyncSearchAction.Request(request.param("id"));
-        return channel -> client.execute(DeleteAsyncSearchAction.INSTANCE, delete, new RestToXContentListener<>(channel));
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
deleted file mode 100644
index 8dc29bf071e0d..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestStatusToXContentListener;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-
-import java.util.List;
-import java.util.Set;
-
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.xpack.search.RestSubmitAsyncSearchAction.RESPONSE_PARAMS;
-
-public class RestGetAsyncSearchAction extends BaseRestHandler  {
-    @Override
-    public List<Route> routes() {
-        return unmodifiableList(asList(new Route(GET, "/_async_search/{id}")));
-    }
-
-
-    @Override
-    public String getName() {
-        return "async_search_get_action";
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(request.param("id"));
-        if (request.hasParam("wait_for_completion")) {
-            get.setWaitForCompletion(request.paramAsTime("wait_for_completion", get.getWaitForCompletion()));
-        }
-        if (request.hasParam("keep_alive")) {
-            get.setKeepAlive(request.paramAsTime("keep_alive", get.getKeepAlive()));
-        }
-        return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel));
-    }
-
-    @Override
-    protected Set<String> responseParams() {
-        return RESPONSE_PARAMS;
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
deleted file mode 100644
index e1020f8e5a234..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestCancellableNodeClient;
-import org.elasticsearch.rest.action.RestStatusToXContentListener;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.function.IntConsumer;
-
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.action.search.RestSearchAction.parseSearchRequest;
-
-public final class RestSubmitAsyncSearchAction extends BaseRestHandler {
-    static final String TYPED_KEYS_PARAM = "typed_keys";
-    static final Set<String> RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM);
-
-    @Override
-    public List<Route> routes() {
-        return unmodifiableList(asList(
-            new Route(POST, "/_async_search"),
-            new Route(POST, "/{index}/_async_search")));
-    }
-
-    @Override
-    public String getName() {
-        return "async_search_submit_action";
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        SubmitAsyncSearchRequest submit = new SubmitAsyncSearchRequest();
-        IntConsumer setSize = size -> submit.getSearchRequest().source().size(size);
-        request.withContentOrSourceParamParserOrNull(parser ->
-            parseSearchRequest(submit.getSearchRequest(), request, parser, setSize));
-
-        if (request.hasParam("wait_for_completion")) {
-            submit.setWaitForCompletion(request.paramAsTime("wait_for_completion", submit.getWaitForCompletion()));
-        }
-        if (request.hasParam("keep_alive")) {
-            submit.setKeepAlive(request.paramAsTime("keep_alive", submit.getKeepAlive()));
-        }
-        if (request.hasParam("clean_on_completion")) {
-            submit.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", submit.isCleanOnCompletion()));
-        }
-        return channel -> {
-            RestStatusToXContentListener<AsyncSearchResponse> listener = new RestStatusToXContentListener<>(channel);
-            RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel());
-            cancelClient.execute(SubmitAsyncSearchAction.INSTANCE, submit, listener);
-        };
-    }
-
-    @Override
-    protected Set<String> responseParams() {
-        return RESPONSE_PARAMS;
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
deleted file mode 100644
index 14bf4eafec517..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionListenerResponseHandler;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-
-import java.io.IOException;
-
-public class TransportDeleteAsyncSearchAction extends HandledTransportAction<DeleteAsyncSearchAction.Request, AcknowledgedResponse> {
-    private final ClusterService clusterService;
-    private final TransportService transportService;
-    private final AsyncSearchIndexService store;
-
-    @Inject
-    public TransportDeleteAsyncSearchAction(TransportService transportService,
-                                            ActionFilters actionFilters,
-                                            ClusterService clusterService,
-                                            ThreadPool threadPool,
-                                            NamedWriteableRegistry registry,
-                                            Client client) {
-        super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new);
-        this.store = new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, registry);
-        this.clusterService = clusterService;
-        this.transportService = transportService;
-    }
-
-    @Override
-    protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, ActionListener<AcknowledgedResponse> listener) {
-        try {
-            AsyncSearchId searchId = AsyncSearchId.decode(request.getId());
-            DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId());
-            if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) {
-                cancelTaskAndDeleteResult(searchId, listener);
-            } else {
-                TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
-                transportService.sendRequest(node, DeleteAsyncSearchAction.NAME, request, builder.build(),
-                    new ActionListenerResponseHandler<>(listener, AcknowledgedResponse::new, ThreadPool.Names.SAME));
-            }
-        } catch (Exception exc) {
-            listener.onFailure(exc);
-        }
-    }
-
-    private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener<AcknowledgedResponse> listener) throws IOException {
-        AsyncSearchTask task = store.getTask(taskManager, searchId);
-        if (task != null) {
-            task.cancelTask(() -> store.deleteResponse(searchId, false, listener));
-        } else {
-            // the task is not running anymore so we throw a not found exception if
-            // the search id is also not present in the index (already deleted) or if the user
-            // is not allowed to access it.
-            store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResponse(searchId, true, listener), listener::onFailure));
-        }
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
deleted file mode 100644
index 01c807b19aea1..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionListenerResponseHandler;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.index.engine.DocumentMissingException;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-
-public class TransportGetAsyncSearchAction extends HandledTransportAction<GetAsyncSearchAction.Request, AsyncSearchResponse> {
-    private final Logger logger = LogManager.getLogger(TransportGetAsyncSearchAction.class);
-    private final ClusterService clusterService;
-    private final TransportService transportService;
-    private final AsyncSearchIndexService store;
-
-    @Inject
-    public TransportGetAsyncSearchAction(TransportService transportService,
-                                         ActionFilters actionFilters,
-                                         ClusterService clusterService,
-                                         NamedWriteableRegistry registry,
-                                         Client client,
-                                         ThreadPool threadPool) {
-        super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncSearchAction.Request::new);
-        this.clusterService = clusterService;
-        this.transportService = transportService;
-        this.store = new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, registry);
-    }
-
-    @Override
-    protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener<AsyncSearchResponse> listener) {
-        try {
-            long nowInMillis = System.currentTimeMillis();
-            AsyncSearchId searchId = AsyncSearchId.decode(request.getId());
-            DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId());
-            if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) {
-                if (request.getKeepAlive().getMillis() > 0) {
-                    long expirationTime = nowInMillis + request.getKeepAlive().getMillis();
-                    store.updateExpirationTime(searchId.getDocId(), expirationTime,
-                        ActionListener.wrap(
-                            p -> getSearchResponseFromTask(searchId, request, nowInMillis, expirationTime, listener),
-                            exc -> {
-                                if (exc.getCause() instanceof DocumentMissingException == false) {
-                                    logger.error("failed to retrieve " + searchId.getEncoded(), exc);
-                                }
-                                listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
-                            }
-                        ));
-                } else {
-                    getSearchResponseFromTask(searchId, request, nowInMillis, -1, listener);
-                }
-            } else {
-                TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
-                transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(),
-                    new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME));
-            }
-        } catch (Exception exc) {
-            listener.onFailure(exc);
-        }
-    }
-
-    private void getSearchResponseFromTask(AsyncSearchId searchId,
-                                           GetAsyncSearchAction.Request request,
-                                           long nowInMillis,
-                                           long expirationTimeMillis,
-                                           ActionListener<AsyncSearchResponse> listener) {
-        try {
-            final AsyncSearchTask task = store.getTask(taskManager, searchId);
-            if (task == null) {
-                getSearchResponseFromIndex(searchId, request, nowInMillis, listener);
-                return;
-            }
-
-            if (task.isCancelled()) {
-                listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
-                return;
-            }
-
-            if (expirationTimeMillis != -1) {
-                task.setExpirationTime(expirationTimeMillis);
-            }
-            task.addCompletionListener(new ActionListener<>() {
-                @Override
-                public void onResponse(AsyncSearchResponse response) {
-                    sendFinalResponse(request, response, nowInMillis, listener);
-                }
-
-                @Override
-                public void onFailure(Exception exc) {
-                    listener.onFailure(exc);
-                }
-            }, request.getWaitForCompletion());
-        } catch (Exception exc) {
-            listener.onFailure(exc);
-        }
-    }
-
-   private void getSearchResponseFromIndex(AsyncSearchId searchId,
-                                           GetAsyncSearchAction.Request request,
-                                           long nowInMillis,
-                                           ActionListener<AsyncSearchResponse> listener) {
-        store.getResponse(searchId, new ActionListener<>() {
-                @Override
-                public void onResponse(AsyncSearchResponse response) {
-                    sendFinalResponse(request, response, nowInMillis, listener);
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    listener.onFailure(e);
-                }
-            });
-    }
-
-    private void sendFinalResponse(GetAsyncSearchAction.Request request,
-                                   AsyncSearchResponse response,
-                                   long nowInMillis,
-                                   ActionListener<AsyncSearchResponse> listener) {
-        // check if the result has expired
-        if (response.getExpirationTime() < nowInMillis) {
-            listener.onFailure(new ResourceNotFoundException(request.getId()));
-            return;
-        }
-
-        listener.onResponse(response);
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
deleted file mode 100644
index a89f166866a9c..0000000000000
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.TransportSearchAction;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.engine.DocumentMissingException;
-import org.elasticsearch.search.SearchService;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
-import org.elasticsearch.tasks.CancellableTask;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskCancelledException;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
-
-import java.util.Map;
-import java.util.function.Supplier;
-
-public class TransportSubmitAsyncSearchAction extends HandledTransportAction<SubmitAsyncSearchRequest, AsyncSearchResponse> {
-    private static final Logger logger = LogManager.getLogger(TransportSubmitAsyncSearchAction.class);
-
-    private final NodeClient nodeClient;
-    private final Supplier<ReduceContext> reduceContextSupplier;
-    private final TransportSearchAction searchAction;
-    private final AsyncSearchIndexService store;
-
-    @Inject
-    public TransportSubmitAsyncSearchAction(ClusterService clusterService,
-                                            TransportService transportService,
-                                            ActionFilters actionFilters,
-                                            NamedWriteableRegistry registry,
-                                            Client client,
-                                            NodeClient nodeClient,
-                                            SearchService searchService,
-                                            TransportSearchAction searchAction) {
-        super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new);
-        this.nodeClient = nodeClient;
-        this.reduceContextSupplier = () -> searchService.createReduceContext(true);
-        this.searchAction = searchAction;
-        this.store = new AsyncSearchIndexService(clusterService, transportService.getThreadPool().getThreadContext(), client, registry);
-    }
-
-    @Override
-    protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener<AsyncSearchResponse> submitListener) {
-        CancellableTask submitTask = (CancellableTask) task;
-        final SearchRequest searchRequest = createSearchRequest(request, submitTask.getId(), request.getKeepAlive());
-        AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest);
-        searchAction.execute(searchTask, searchRequest, searchTask.getSearchProgressActionListener());
-        searchTask.addCompletionListener(
-            new ActionListener<>() {
-                @Override
-                public void onResponse(AsyncSearchResponse searchResponse) {
-                    if (searchResponse.isRunning() || request.isCleanOnCompletion() == false) {
-                        // the task is still running and the user cannot wait more so we create
-                        // a document for further retrieval
-                        try {
-                            if (submitTask.isCancelled()) {
-                                // the user cancelled the submit so we don't store anything
-                                // and propagate the failure
-                                Exception cause = new TaskCancelledException(submitTask.getReasonCancelled());
-                                onFatalFailure(searchTask, cause, false, submitListener);
-                            } else {
-                                final String docId = searchTask.getSearchId().getDocId();
-                                // creates the fallback response if the node crashes/restarts in the middle of the request
-                                // TODO: store intermediate results ?
-                                AsyncSearchResponse initialResp = searchResponse.clone(searchResponse.getId());
-                                store.storeInitialResponse(docId, searchTask.getOriginHeaders(), initialResp,
-                                    new ActionListener<>() {
-                                        @Override
-                                        public void onResponse(IndexResponse r) {
-                                            if (searchResponse.isRunning()) {
-                                                try {
-                                                    // store the final response on completion unless the submit is cancelled
-                                                    searchTask.addCompletionListener(finalResponse ->
-                                                        onFinalResponse(submitTask, searchTask, finalResponse, () -> {}));
-                                                } finally {
-                                                    submitListener.onResponse(searchResponse);
-                                                }
-                                            } else {
-                                                onFinalResponse(submitTask, searchTask, searchResponse,
-                                                    () -> submitListener.onResponse(searchResponse));
-                                            }
-                                        }
-
-                                        @Override
-                                        public void onFailure(Exception exc) {
-                                            onFatalFailure(searchTask, exc, searchResponse.isRunning(), submitListener);
-                                        }
-                                    });
-                            }
-                        } catch (Exception exc) {
-                            onFatalFailure(searchTask, exc, searchResponse.isRunning(), submitListener);
-                        }
-                    } else {
-                        // the task completed within the timeout so the response is sent back to the user
-                        // with a null id since nothing was stored on the cluster.
-                        taskManager.unregister(searchTask);
-                        submitListener.onResponse(searchResponse.clone(null));
-                    }
-                }
-
-                @Override
-                public void onFailure(Exception exc) {
-                    submitListener.onFailure(exc);
-                }
-            }, request.getWaitForCompletion());
-    }
-
-    private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, long parentTaskId, TimeValue keepAlive) {
-        String docID = UUIDs.randomBase64UUID();
-        Map<String, String> originHeaders = nodeClient.threadPool().getThreadContext().getHeaders();
-        SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) {
-            @Override
-            public AsyncSearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> taskHeaders) {
-                AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id));
-                return new AsyncSearchTask(id, type, action, parentTaskId, keepAlive, originHeaders, taskHeaders, searchId,
-                    store.getClient(), nodeClient.threadPool(), reduceContextSupplier);
-            }
-        };
-        searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), parentTaskId));
-        return searchRequest;
-    }
-
-    private void onFatalFailure(AsyncSearchTask task, Exception error, boolean shouldCancel, ActionListener<AsyncSearchResponse> listener) {
-        if (shouldCancel) {
-            task.cancelTask(() -> {
-                try {
-                    task.addCompletionListener(finalResponse -> taskManager.unregister(task));
-                } finally {
-                    listener.onFailure(error);
-                }
-            });
-        } else {
-            try {
-                task.addCompletionListener(finalResponse -> taskManager.unregister(task));
-            } finally {
-                listener.onFailure(error);
-            }
-        }
-    }
-
-    private void onFinalResponse(CancellableTask submitTask,
-                                 AsyncSearchTask searchTask,
-                                 AsyncSearchResponse response,
-                                 Runnable nextAction) {
-        if (submitTask.isCancelled() || searchTask.isCancelled()) {
-            // the user cancelled the submit so we ensure that there is nothing stored in the response index.
-            store.deleteResponse(searchTask.getSearchId(), false, ActionListener.wrap(() -> {
-                taskManager.unregister(searchTask);
-                nextAction.run();
-            }));
-            return;
-        }
-
-        try {
-            store.storeFinalResponse(searchTask.getSearchId().getDocId(), response, new ActionListener<>() {
-                @Override
-                public void onResponse(UpdateResponse updateResponse) {
-                    taskManager.unregister(searchTask);
-                    nextAction.run();
-                }
-
-                @Override
-                public void onFailure(Exception exc) {
-                    if (exc.getCause() instanceof DocumentMissingException == false) {
-                        logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]",
-                            searchTask.getSearchId().getEncoded()), exc);
-                    }
-                    taskManager.unregister(searchTask);
-                    nextAction.run();
-                }
-            });
-        } catch (Exception exc) {
-            logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", searchTask.getSearchId().getEncoded()), exc);
-            taskManager.unregister(searchTask);
-            nextAction.run();
-        }
-    }
-}
diff --git a/x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
deleted file mode 100644
index 373030fe12efa..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
-import org.elasticsearch.search.aggregations.metrics.InternalMax;
-import org.elasticsearch.search.aggregations.metrics.InternalMin;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
-import org.junit.Before;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-
-// TODO: add tests for keepAlive and expiration
-public class AsyncSearchActionTests extends AsyncSearchIntegTestCase {
-    private String indexName;
-    private int numShards;
-
-    private int numKeywords;
-    private Map<String, AtomicInteger> keywordFreqs;
-    private float maxMetric = Float.NEGATIVE_INFINITY;
-    private float minMetric = Float.POSITIVE_INFINITY;
-
-    @Before
-    public void indexDocuments() throws InterruptedException {
-        indexName = "test-async";
-        numShards = randomIntBetween(internalCluster().numDataNodes(), internalCluster().numDataNodes()*10);
-        int numDocs = randomIntBetween(numShards, numShards*3);
-        createIndex(indexName, Settings.builder().put("index.number_of_shards", numShards).build());
-        numKeywords = randomIntBetween(1, 100);
-        keywordFreqs = new HashMap<>();
-        Set<String> keywordSet = new HashSet<>();
-        for (int i = 0; i < numKeywords; i++) {
-            keywordSet.add(randomAlphaOfLengthBetween(10, 20));
-        }
-        numKeywords = keywordSet.size();
-        String[] keywords = keywordSet.toArray(String[]::new);
-        List<IndexRequestBuilder> reqs = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            float metric = randomFloat();
-            maxMetric = Math.max(metric, maxMetric);
-            minMetric = Math.min(metric, minMetric);
-            String keyword = keywords[randomIntBetween(0, numKeywords-1)];
-            keywordFreqs.compute(keyword,
-                (k, v) -> {
-                    if (v == null) {
-                        return new AtomicInteger(1);
-                    }
-                    v.incrementAndGet();
-                    return v;
-            });
-            reqs.add(client().prepareIndex(indexName).setSource("terms", keyword, "metric", metric));
-        }
-        indexRandom(true, true, reqs);
-        ensureGreen("test-async");
-    }
-
-    public void testMaxMinAggregation() throws Exception {
-        int step = numShards > 2 ? randomIntBetween(2, numShards) : 2;
-        int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0;
-        SearchSourceBuilder source = new SearchSourceBuilder()
-            .aggregation(AggregationBuilders.min("min").field("metric"))
-            .aggregation(AggregationBuilders.max("max").field("metric"));
-        try (SearchResponseIterator it =
-                 assertBlockingIterator(indexName, source, numFailures, step)) {
-            AsyncSearchResponse response = it.next();
-            while (it.hasNext()) {
-                response = it.next();
-                assertNotNull(response.getSearchResponse());
-                if (response.getSearchResponse().getSuccessfulShards() > 0) {
-                    assertNotNull(response.getSearchResponse().getAggregations());
-                    assertNotNull(response.getSearchResponse().getAggregations().get("max"));
-                    assertNotNull(response.getSearchResponse().getAggregations().get("min"));
-                    InternalMax max = response.getSearchResponse().getAggregations().get("max");
-                    InternalMin min = response.getSearchResponse().getAggregations().get("min");
-                    assertThat((float) min.getValue(), greaterThanOrEqualTo(minMetric));
-                    assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric));
-                }
-            }
-            if (numFailures == numShards) {
-                assertNotNull(response.getFailure());
-            } else {
-                assertNotNull(response.getSearchResponse());
-                assertNotNull(response.getSearchResponse().getAggregations());
-                assertNotNull(response.getSearchResponse().getAggregations().get("max"));
-                assertNotNull(response.getSearchResponse().getAggregations().get("min"));
-                InternalMax max = response.getSearchResponse().getAggregations().get("max");
-                InternalMin min = response.getSearchResponse().getAggregations().get("min");
-                if (numFailures == 0) {
-                    assertThat((float) min.getValue(), equalTo(minMetric));
-                    assertThat((float) max.getValue(), equalTo(maxMetric));
-                } else {
-                    assertThat((float) min.getValue(), greaterThanOrEqualTo(minMetric));
-                    assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric));
-                }
-            }
-            deleteAsyncSearch(response.getId());
-            ensureTaskRemoval(response.getId());
-        }
-    }
-
-    public void testTermsAggregation() throws Exception {
-        int step = numShards > 2 ? randomIntBetween(2, numShards) : 2;
-        int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0;
-        SearchSourceBuilder source = new SearchSourceBuilder()
-            .aggregation(AggregationBuilders.terms("terms").field("terms.keyword").size(numKeywords));
-        try (SearchResponseIterator it =
-                 assertBlockingIterator(indexName, source, numFailures, step)) {
-            AsyncSearchResponse response = it.next();
-            while (it.hasNext()) {
-                response = it.next();
-                assertNotNull(response.getSearchResponse());
-                if (response.getSearchResponse().getSuccessfulShards() > 0) {
-                    assertNotNull(response.getSearchResponse().getAggregations());
-                    assertNotNull(response.getSearchResponse().getAggregations().get("terms"));
-                    StringTerms terms = response.getSearchResponse().getAggregations().get("terms");
-                    assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0));
-                    assertThat(terms.getBuckets().size(), lessThanOrEqualTo(numKeywords));
-                    for (InternalTerms.Bucket<?> bucket : terms.getBuckets()) {
-                        long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get();
-                        assertThat(bucket.getDocCount(), lessThanOrEqualTo(count));
-                    }
-                }
-            }
-            if (numFailures == numShards) {
-                assertNotNull(response.getFailure());
-            } else {
-                assertNotNull(response.getSearchResponse());
-                assertNotNull(response.getSearchResponse().getAggregations());
-                assertNotNull(response.getSearchResponse().getAggregations().get("terms"));
-                StringTerms terms = response.getSearchResponse().getAggregations().get("terms");
-                assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0));
-                assertThat(terms.getBuckets().size(), lessThanOrEqualTo(numKeywords));
-                for (InternalTerms.Bucket<?> bucket : terms.getBuckets()) {
-                    long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get();
-                    if (numFailures > 0) {
-                        assertThat(bucket.getDocCount(), lessThanOrEqualTo(count));
-                    } else {
-                        assertThat(bucket.getDocCount(), equalTo(count));
-                    }
-                }
-            }
-            deleteAsyncSearch(response.getId());
-            ensureTaskRemoval(response.getId());
-        }
-    }
-
-    public void testRestartAfterCompletion() throws Exception {
-        final AsyncSearchResponse initial;
-        try (SearchResponseIterator it =
-                 assertBlockingIterator(indexName, new SearchSourceBuilder(), 0, 2)) {
-            initial = it.next();
-        }
-        ensureTaskCompletion(initial.getId());
-        restartTaskNode(initial.getId());
-        AsyncSearchResponse response = getAsyncSearch(initial.getId());
-        assertNotNull(response.getSearchResponse());
-        assertFalse(response.isRunning());
-        assertFalse(response.isPartial());
-        deleteAsyncSearch(response.getId());
-        ensureTaskRemoval(response.getId());
-    }
-
-    public void testDeleteCancelRunningTask() throws Exception {
-        final AsyncSearchResponse initial;
-        SearchResponseIterator it =
-            assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2);
-        initial = it.next();
-        deleteAsyncSearch(initial.getId());
-        it.close();
-        ensureTaskCompletion(initial.getId());
-        ensureTaskRemoval(initial.getId());
-    }
-
-    public void testDeleteCleanupIndex() throws Exception {
-        SearchResponseIterator it =
-            assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2);
-        AsyncSearchResponse response = it.next();
-        deleteAsyncSearch(response.getId());
-        it.close();
-        ensureTaskCompletion(response.getId());
-        ensureTaskRemoval(response.getId());
-    }
-
-    public void testCleanupOnFailure() throws Exception {
-        final AsyncSearchResponse initial;
-        try (SearchResponseIterator it =
-                 assertBlockingIterator(indexName, new SearchSourceBuilder(), numShards, 2)) {
-            initial = it.next();
-        }
-        ensureTaskCompletion(initial.getId());
-        AsyncSearchResponse response = getAsyncSearch(initial.getId());
-        assertFalse(response.isRunning());
-        assertNotNull(response.getFailure());
-        assertTrue(response.isPartial());
-        assertThat(response.getSearchResponse().getTotalShards(), equalTo(numShards));
-        assertThat(response.getSearchResponse().getShardFailures().length, equalTo(numShards));
-        deleteAsyncSearch(initial.getId());
-        ensureTaskRemoval(initial.getId());
-    }
-
-    public void testInvalidId() throws Exception {
-        SearchResponseIterator it =
-            assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2);
-        AsyncSearchResponse response = it.next();
-        ExecutionException exc = expectThrows(ExecutionException.class, () -> getAsyncSearch("invalid"));
-        assertThat(exc.getCause(), instanceOf(IllegalArgumentException.class));
-        assertThat(exc.getMessage(), containsString("invalid id"));
-        while (it.hasNext()) {
-            response = it.next();
-        }
-        assertFalse(response.isRunning());
-    }
-
-    public void testNoIndex() throws Exception {
-        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest("invalid-*");
-        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
-        AsyncSearchResponse response = submitAsyncSearch(request);
-        assertNotNull(response.getSearchResponse());
-        assertFalse(response.isRunning());
-        assertThat(response.getSearchResponse().getTotalShards(), equalTo(0));
-
-        request = new SubmitAsyncSearchRequest("invalid");
-        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
-        response = submitAsyncSearch(request);
-        assertNull(response.getSearchResponse());
-        assertNotNull(response.getFailure());
-        assertFalse(response.isRunning());
-        ElasticsearchException exc = response.getFailure();
-        assertThat(exc.getMessage(), containsString("no such index"));
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java
deleted file mode 100644
index ebfa4bfc97190..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.test.ESTestCase;
-
-public class AsyncSearchIdTests extends ESTestCase {
-    public void testEncode() {
-        for (int i = 0; i < 10; i++) {
-            AsyncSearchId instance = new AsyncSearchId(UUIDs.randomBase64UUID(),
-                new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong()));
-            String encoded = AsyncSearchId.encode(instance.getDocId(), instance.getTaskId());
-            AsyncSearchId same = AsyncSearchId.decode(encoded);
-            assertEquals(same, instance);
-
-            AsyncSearchId mutate = mutate(instance);
-            assertNotEquals(mutate, instance);
-            assertNotEquals(mutate, same);
-        }
-    }
-
-    private AsyncSearchId mutate(AsyncSearchId id) {
-        int rand = randomIntBetween(0, 1);
-        switch (rand) {
-            case 0:
-                return new AsyncSearchId(randomAlphaOfLength(id.getDocId().length()+1), id.getTaskId());
-
-            case 1:
-                return new AsyncSearchId(id.getDocId(),
-                    new TaskId(randomAlphaOfLength(id.getTaskId().getNodeId().length()), randomNonNegativeLong()));
-
-            default:
-                throw new AssertionError();
-        }
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
deleted file mode 100644
index afc7627b2e4df..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.user.User;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.Collections;
-
-import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses;
-import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse;
-import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse;
-import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId;
-
-// TODO: test CRUD operations
-public class AsyncSearchIndexServiceTests extends ESSingleNodeTestCase {
-    private AsyncSearchIndexService indexService;
-
-    @Before
-    public void setup() {
-        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
-        TransportService transportService = getInstanceFromNode(TransportService.class);
-        indexService = new AsyncSearchIndexService(clusterService, transportService.getThreadPool().getThreadContext(),
-            client(), writableRegistry());
-    }
-
-    public void testEncodeSearchResponse() throws IOException {
-        for (int i = 0; i < 10; i++) {
-            AsyncSearchResponse response = randomAsyncSearchResponse(randomSearchId(), randomSearchResponse());
-            String encoded = indexService.encodeResponse(response);
-            AsyncSearchResponse same = indexService.decodeResponse(encoded);
-            assertEqualResponses(response, same);
-        }
-    }
-
-    public void testEnsuredAuthenticatedUserIsSame() throws IOException {
-        Authentication original =
-            new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null);
-        Authentication current = randomBoolean() ? original :
-            new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null);
-        assertTrue(indexService.ensureAuthenticatedUserIsSame(original, current));
-        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        original.writeToContext(threadContext);
-        assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current));
-
-        // original is not set
-        assertTrue(indexService.ensureAuthenticatedUserIsSame(Collections.emptyMap(), current));
-        // current is not set
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), null));
-
-        // original user being run as
-        User user = new User(new User("test", "role"), new User("authenticated", "runas"));
-        current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"),
-            new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
-        assertTrue(indexService.ensureAuthenticatedUserIsSame(original, current));
-        assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current));
-
-        // both user are run as
-        current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"),
-            new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
-        Authentication runAs = current;
-        assertTrue(indexService.ensureAuthenticatedUserIsSame(runAs, current));
-        threadContext = new ThreadContext(Settings.EMPTY);
-        original.writeToContext(threadContext);
-        assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current));
-
-        // different authenticated by type
-        Authentication differentRealmType =
-            new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", randomAlphaOfLength(5), "node"), null);
-        threadContext = new ThreadContext(Settings.EMPTY);
-        original.writeToContext(threadContext);
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, differentRealmType));
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), differentRealmType));
-
-        // wrong user
-        Authentication differentUser =
-            new Authentication(new User("test2", "role"), new Authentication.RealmRef("realm", "realm", "node"), null);
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, differentUser));
-
-        // run as different user
-        Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")),
-            new Authentication.RealmRef("realm", "file", "node1"), new Authentication.RealmRef("realm", "file", "node1"));
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, diffRunAs));
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), diffRunAs));
-
-        // run as different looked up by type
-        Authentication runAsDiffType = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"),
-            new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node"));
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, runAsDiffType));
-        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), runAsDiffType));
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
deleted file mode 100644
index e5e65a832db8e..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.TotalHits;
-import org.apache.lucene.search.Weight;
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
-import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
-import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.reindex.ReindexPlugin;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.plugins.SearchPlugin;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
-import org.elasticsearch.xpack.ilm.IndexLifecycle;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.xpack.search.AsyncSearchIndexService.INDEX;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-
-public abstract class AsyncSearchIntegTestCase extends ESIntegTestCase {
-    interface SearchResponseIterator extends Iterator<AsyncSearchResponse>, Closeable {}
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class,
-            QueryBlockPlugin.class, ReindexPlugin.class);
-    }
-
-    /**
-     * Restart the node that runs the {@link TaskId} decoded from the provided {@link AsyncSearchId}.
-     */
-    protected void restartTaskNode(String id) throws Exception {
-        AsyncSearchId searchId = AsyncSearchId.decode(id);
-        final ClusterStateResponse clusterState = client().admin().cluster()
-            .prepareState().clear().setNodes(true).get();
-        DiscoveryNode node = clusterState.getState().nodes().get(searchId.getTaskId().getNodeId());
-        internalCluster().restartNode(node.getName(), new InternalTestCluster.RestartCallback() {
-            @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                return super.onNodeStopped(nodeName);
-            }
-        });
-        ensureYellow(INDEX);
-    }
-
-    protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request) throws ExecutionException, InterruptedException {
-        return client().execute(SubmitAsyncSearchAction.INSTANCE, request).get();
-    }
-
-    protected AsyncSearchResponse getAsyncSearch(String id) throws ExecutionException, InterruptedException {
-        return client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncSearchAction.Request(id)).get();
-    }
-
-    protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionException, InterruptedException {
-        return client().execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(id)).get();
-    }
-
-    /**
-     * Wait the removal of the document decoded from the provided {@link AsyncSearchId}.
-     */
-    protected void ensureTaskRemoval(String id) throws Exception {
-        AsyncSearchId searchId = AsyncSearchId.decode(id);
-        assertBusy(() -> {
-            GetResponse resp = client().prepareGet()
-                .setIndex(INDEX)
-                .setId(searchId.getDocId())
-                .get();
-            assertFalse(resp.isExists());
-        });
-    }
-
-    /**
-     * Wait the completion of the {@link TaskId} decoded from the provided {@link AsyncSearchId}.
-     */
-    protected void ensureTaskCompletion(String id) throws Exception {
-        assertBusy(() -> {
-            TaskId taskId = AsyncSearchId.decode(id).getTaskId();
-            try {
-                GetTaskResponse resp = client().admin().cluster()
-                    .prepareGetTask(taskId).get();
-                assertNull(resp.getTask());
-            } catch (Exception exc) {
-                if (exc.getCause() instanceof ResourceNotFoundException == false) {
-                    throw exc;
-                }
-            }
-        });
-    }
-
-    protected SearchResponseIterator assertBlockingIterator(String indexName,
-                                                            SearchSourceBuilder source,
-                                                            int numFailures,
-                                                            int progressStep) throws Exception {
-        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(source, indexName);
-        request.setBatchedReduceSize(progressStep);
-        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
-        ClusterSearchShardsResponse response = dataNodeClient().admin().cluster()
-            .prepareSearchShards(request.getSearchRequest().indices()).get();
-        AtomicInteger failures = new AtomicInteger(numFailures);
-        Map<ShardId, ShardIdLatch> shardLatchMap = Arrays.stream(response.getGroups())
-            .map(ClusterSearchShardsGroup::getShardId)
-            .collect(
-                Collectors.toMap(
-                    Function.identity(),
-                    id -> new ShardIdLatch(id, new CountDownLatch(1), failures.decrementAndGet() >= 0)
-                )
-            );
-        ShardIdLatch[] shardLatchArray = shardLatchMap.values().stream()
-            .sorted(Comparator.comparing(ShardIdLatch::shard))
-            .toArray(ShardIdLatch[]::new);
-        resetPluginsLatch(shardLatchMap);
-        request.getSearchRequest().source().query(new BlockQueryBuilder(shardLatchMap));
-
-        final AsyncSearchResponse initial = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get();
-
-        assertTrue(initial.isPartial());
-        assertThat(initial.status(), equalTo(RestStatus.OK));
-        assertThat(initial.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
-        assertThat(initial.getSearchResponse().getSuccessfulShards(), equalTo(0));
-        assertThat(initial.getSearchResponse().getShardFailures().length, equalTo(0));
-
-        return new SearchResponseIterator() {
-            private AsyncSearchResponse response = initial;
-            private int shardIndex = 0;
-            private boolean isFirst = true;
-
-            @Override
-            public boolean hasNext() {
-                return response.isRunning();
-            }
-
-            @Override
-            public AsyncSearchResponse next() {
-                try {
-                    return doNext();
-                } catch (Exception e) {
-                    throw new RuntimeException(e);
-                }
-            }
-
-            private AsyncSearchResponse doNext() throws Exception {
-                if (isFirst) {
-                    isFirst = false;
-                    return response;
-                }
-                AtomicReference<AsyncSearchResponse> atomic = new AtomicReference<>();
-                int step = shardIndex == 0 ? progressStep+1 : progressStep-1;
-                int index = 0;
-                while (index < step && shardIndex < shardLatchArray.length) {
-                    if (shardLatchArray[shardIndex].shouldFail == false) {
-                        ++index;
-                    }
-                    shardLatchArray[shardIndex++].countDown();
-                }
-                AsyncSearchResponse newResponse = client().execute(GetAsyncSearchAction.INSTANCE,
-                    new GetAsyncSearchAction.Request(response.getId())
-                        .setWaitForCompletion(TimeValue.timeValueMillis(10))).get();
-
-                if (newResponse.isRunning()) {
-                    assertThat(newResponse.status(),  equalTo(RestStatus.OK));
-                    assertTrue(newResponse.isPartial());
-                    assertNull(newResponse.getFailure());
-                    assertNotNull(newResponse.getSearchResponse());
-                    assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
-                    assertThat(newResponse.getSearchResponse().getShardFailures().length, lessThanOrEqualTo(numFailures));
-                } else if (numFailures == shardLatchArray.length) {
-                    assertThat(newResponse.status(),  equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-                    assertNotNull(newResponse.getFailure());
-                    assertTrue(newResponse.isPartial());
-                    assertNotNull(newResponse.getSearchResponse());
-                    assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
-                    assertThat(newResponse.getSearchResponse().getSuccessfulShards(), equalTo(0));
-                    assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures));
-                    assertNull(newResponse.getSearchResponse().getAggregations());
-                    assertNotNull(newResponse.getSearchResponse().getHits().getTotalHits());
-                    assertThat(newResponse.getSearchResponse().getHits().getTotalHits().value, equalTo(0L));
-                    assertThat(newResponse.getSearchResponse().getHits().getTotalHits().relation,
-                        equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
-                } else {
-                    assertThat(newResponse.status(),  equalTo(RestStatus.OK));
-                    assertNotNull(newResponse.getSearchResponse());
-                    assertFalse(newResponse.isPartial());
-                    assertThat(newResponse.status(), equalTo(RestStatus.OK));
-                    assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
-                    assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures));
-                    assertThat(newResponse.getSearchResponse().getSuccessfulShards(),
-                        equalTo(shardLatchArray.length-newResponse.getSearchResponse().getShardFailures().length));
-                }
-                return response = newResponse;
-            }
-
-            @Override
-            public void close() {
-                Arrays.stream(shardLatchArray).forEach(shard -> {
-                    if (shard.latch.getCount() == 1) {
-                        shard.latch.countDown();
-                    }
-                });
-            }
-        };
-    }
-
-    private void resetPluginsLatch(Map<ShardId, ShardIdLatch> newLatch) {
-        for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) {
-            pluginsService.filterPlugins(QueryBlockPlugin.class).forEach(p -> p.reset(newLatch));
-        }
-    }
-
-    public static class QueryBlockPlugin extends Plugin implements SearchPlugin {
-        private Map<ShardId, ShardIdLatch> shardsLatch;
-
-        public QueryBlockPlugin() {
-            this.shardsLatch = null;
-        }
-
-        public void reset(Map<ShardId, ShardIdLatch> newLatch) {
-            shardsLatch = newLatch;
-        }
-
-        @Override
-        public List<QuerySpec<?>> getQueries() {
-            return Collections.singletonList(
-                new QuerySpec<>("block_match_all",
-                    in -> new BlockQueryBuilder(in, shardsLatch),
-                    p -> BlockQueryBuilder.fromXContent(p, shardsLatch))
-            );
-        }
-    }
-
-    private static class BlockQueryBuilder extends AbstractQueryBuilder<BlockQueryBuilder> {
-        public static final String NAME = "block_match_all";
-        private final Map<ShardId, ShardIdLatch> shardsLatch;
-
-        private BlockQueryBuilder(Map<ShardId, ShardIdLatch> shardsLatch) {
-            super();
-            this.shardsLatch = shardsLatch;
-        }
-
-        BlockQueryBuilder(StreamInput in, Map<ShardId, ShardIdLatch> shardsLatch) throws IOException {
-            super(in);
-            this.shardsLatch = shardsLatch;
-        }
-
-        private BlockQueryBuilder() {
-            this.shardsLatch = null;
-        }
-
-        @Override
-        protected void doWriteTo(StreamOutput out) {}
-
-        @Override
-        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME);
-            builder.endObject();
-        }
-
-        private static final ObjectParser<BlockQueryBuilder, Void> PARSER = new ObjectParser<>(NAME, BlockQueryBuilder::new);
-
-        public static BlockQueryBuilder fromXContent(XContentParser parser, Map<ShardId, ShardIdLatch> shardsLatch) {
-            try {
-                PARSER.apply(parser, null);
-                return new BlockQueryBuilder(shardsLatch);
-            } catch (IllegalArgumentException e) {
-                throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
-            }
-        }
-
-        @Override
-        protected Query doToQuery(QueryShardContext context) {
-            final Query delegate = Queries.newMatchAllQuery();
-            return new Query() {
-                @Override
-                public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
-                    if (shardsLatch != null) {
-                        try {
-                            final ShardIdLatch latch = shardsLatch.get(new ShardId(context.index(), context.getShardId()));
-                            latch.await();
-                            if (latch.shouldFail) {
-                                throw new IOException("boum");
-                            }
-                        } catch (InterruptedException e) {
-                            throw new RuntimeException(e);
-                        }
-                    }
-                    return delegate.createWeight(searcher, scoreMode, boost);
-                }
-
-                @Override
-                public String toString(String field) {
-                    return delegate.toString(field);
-                }
-
-                @Override
-                public boolean equals(Object obj) {
-                    return false;
-                }
-
-                @Override
-                public int hashCode() {
-                    return 0;
-                }
-            };
-        }
-
-        @Override
-        protected boolean doEquals(BlockQueryBuilder other) {
-            return false;
-        }
-
-        @Override
-        protected int doHashCode() {
-            return 0;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-    }
-
-    private static class ShardIdLatch {
-        private final ShardId shard;
-        private final CountDownLatch latch;
-        private final boolean shouldFail;
-
-        private ShardIdLatch(ShardId shard, CountDownLatch latch, boolean shouldFail) {
-            this.shard = shard;
-            this.latch = latch;
-            this.shouldFail = shouldFail;
-        }
-
-        ShardId shard() {
-            return shard;
-        }
-
-        void countDown() {
-            latch.countDown();
-        }
-
-        void await() throws InterruptedException {
-            latch.await();
-        }
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
deleted file mode 100644
index 925ce9d7897eb..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.elasticsearch.xpack.core.transform.TransformField;
-import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider;
-import org.elasticsearch.xpack.core.transform.transforms.SyncConfig;
-import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.List;
-
-import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId;
-
-public class AsyncSearchResponseTests extends ESTestCase {
-    private SearchResponse searchResponse = randomSearchResponse();
-    private NamedWriteableRegistry namedWriteableRegistry;
-
-    @Before
-    public void registerNamedObjects() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
-
-        List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
-        namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(),
-            TimeSyncConfig::new));
-
-        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
-        namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers());
-
-        namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
-    }
-
-
-    protected Writeable.Reader<AsyncSearchResponse> instanceReader() {
-        return AsyncSearchResponse::new;
-    }
-
-    protected AsyncSearchResponse createTestInstance() {
-        return randomAsyncSearchResponse(randomSearchId(), searchResponse);
-    }
-
-    protected void assertEqualInstances(AsyncSearchResponse expectedInstance, AsyncSearchResponse newInstance) {
-        assertNotSame(newInstance, expectedInstance);
-        assertEqualResponses(expectedInstance, newInstance);
-    }
-
-    public final void testSerialization() throws IOException {
-        for (int runs = 0; runs < 10; runs++) {
-            AsyncSearchResponse testInstance = createTestInstance();
-            assertSerialization(testInstance);
-        }
-    }
-
-    protected final AsyncSearchResponse assertSerialization(AsyncSearchResponse testInstance) throws IOException {
-        return assertSerialization(testInstance, Version.CURRENT);
-    }
-
-    protected final AsyncSearchResponse assertSerialization(AsyncSearchResponse testInstance, Version version) throws IOException {
-        AsyncSearchResponse deserializedInstance = copyInstance(testInstance, version);
-        assertEqualInstances(testInstance, deserializedInstance);
-        return deserializedInstance;
-    }
-
-    protected final AsyncSearchResponse copyInstance(AsyncSearchResponse instance) throws IOException {
-        return copyInstance(instance, Version.CURRENT);
-    }
-
-    protected AsyncSearchResponse copyInstance(AsyncSearchResponse instance, Version version) throws IOException {
-        return copyWriteable(instance, namedWriteableRegistry, instanceReader(), version);
-    }
-
-    static AsyncSearchResponse randomAsyncSearchResponse(String searchId, SearchResponse searchResponse) {
-        int rand = randomIntBetween(0, 2);
-        switch (rand) {
-            case 0:
-                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean(),
-                    randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong());
-
-            case 1:
-                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse, null,
-                    randomBoolean(), randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong());
-
-            case 2:
-                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse,
-                    new ElasticsearchException(new IOException("boum")), randomBoolean(), randomBoolean(),
-                    randomNonNegativeLong(), randomNonNegativeLong());
-
-            default:
-                throw new AssertionError();
-        }
-    }
-
-    static SearchResponse randomSearchResponse() {
-        long tookInMillis = randomNonNegativeLong();
-        int totalShards = randomIntBetween(1, Integer.MAX_VALUE);
-        int successfulShards = randomIntBetween(0, totalShards);
-        int skippedShards = totalShards - successfulShards;
-        InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
-        return new SearchResponse(internalSearchResponse, null, totalShards,
-            successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
-    }
-
-    static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchResponse actual) {
-        assertEquals(expected.getId(), actual.getId());
-        assertEquals(expected.getVersion(), actual.getVersion());
-        assertEquals(expected.status(), actual.status());
-        assertEquals(expected.getFailure() == null, actual.getFailure() == null);
-        assertEquals(expected.isRunning(), actual.isRunning());
-        assertEquals(expected.isPartial(), actual.isPartial());
-        assertEquals(expected.getStartTime(), actual.getStartTime());
-        assertEquals(expected.getExpirationTime(), actual.getExpirationTime());
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
deleted file mode 100644
index 8c3f57883ec14..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchShard;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.client.NoOpClient;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
-import org.junit.After;
-import org.junit.Before;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class AsyncSearchTaskTests extends ESTestCase {
-    private ThreadPool threadPool;
-
-    @Before
-    public void beforeTest() {
-        threadPool = new TestThreadPool(getTestName());
-    }
-
-    @After
-    public void afterTest() {
-        threadPool.shutdownNow();
-    }
-
-    public void testWaitForInit() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
-            Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
-            new NoOpClient(threadPool), threadPool, null);
-        int numShards = randomIntBetween(0, 10);
-        List<SearchShard> shards = new ArrayList<>();
-        for (int i = 0; i < numShards; i++) {
-            shards.add(new SearchShard(null, new ShardId("0", "0", 1)));
-        }
-        List<SearchShard> skippedShards = new ArrayList<>();
-        int numSkippedShards = randomIntBetween(0, 10);
-        for (int i = 0; i < numSkippedShards; i++) {
-            skippedShards.add(new SearchShard(null, new ShardId("0", "0", 1)));
-        }
-
-        int numThreads = randomIntBetween(1, 10);
-        CountDownLatch latch = new CountDownLatch(numThreads);
-        for (int i = 0; i < numThreads; i++) {
-            Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() {
-                @Override
-                public void onResponse(AsyncSearchResponse resp) {
-                    assertThat(numShards + numSkippedShards, equalTo(resp.getSearchResponse().getTotalShards()));
-                    assertThat(numSkippedShards, equalTo(resp.getSearchResponse().getSkippedShards()));
-                    assertThat(0, equalTo(resp.getSearchResponse().getFailedShards()));
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    throw new AssertionError(e);
-
-                }
-            }, TimeValue.timeValueMillis(1)));
-            thread.start();
-        }
-        assertFalse(latch.await(numThreads*2, TimeUnit.MILLISECONDS));
-        task.getSearchProgressActionListener().onListShards(shards, skippedShards, SearchResponse.Clusters.EMPTY, false);
-        latch.await();
-    }
-
-    public void testWithFailure() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
-            Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
-            new NoOpClient(threadPool), threadPool, null);
-        int numThreads = randomIntBetween(1, 10);
-        CountDownLatch latch = new CountDownLatch(numThreads);
-        for (int i = 0; i < numThreads; i++) {
-            Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() {
-                @Override
-                public void onResponse(AsyncSearchResponse resp) {
-                    assertNull(resp.getSearchResponse());
-                    assertNotNull(resp.getFailure());
-                    assertTrue(resp.isPartial());
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    throw new AssertionError(e);
-                }
-            }, TimeValue.timeValueMillis(1)));
-            thread.start();
-        }
-        assertFalse(latch.await(numThreads*2, TimeUnit.MILLISECONDS));
-        task.getSearchProgressActionListener().onFailure(new Exception("boom"));
-        latch.await();
-    }
-
-    public void testWaitForCompletion() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
-            Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
-            new NoOpClient(threadPool), threadPool, null);
-        int numShards = randomIntBetween(0, 10);
-        List<SearchShard> shards = new ArrayList<>();
-        for (int i = 0; i < numShards; i++) {
-            shards.add(new SearchShard(null, new ShardId("0", "0", 1)));
-        }
-        List<SearchShard> skippedShards = new ArrayList<>();
-        int numSkippedShards = randomIntBetween(0, 10);
-        for (int i = 0; i < numSkippedShards; i++) {
-            skippedShards.add(new SearchShard(null, new ShardId("0", "0", 1)));
-        }
-
-        int numShardFailures = 0;
-        task.getSearchProgressActionListener().onListShards(shards, skippedShards, SearchResponse.Clusters.EMPTY, false);
-        for (int i = 0; i < numShards; i++) {
-            task.getSearchProgressActionListener().onPartialReduce(shards.subList(i, i+1),
-                new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), null, 0);
-            assertCompletionListeners(task, numShards+numSkippedShards, numSkippedShards, numShardFailures, true);
-        }
-        task.getSearchProgressActionListener().onFinalReduce(shards,
-            new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), null, 0);
-        assertCompletionListeners(task, numShards+numSkippedShards, numSkippedShards, numShardFailures, true);
-        ((AsyncSearchTask.Listener)task.getProgressListener()).onResponse(
-            newSearchResponse(numShards+numSkippedShards, numShards, numSkippedShards));
-        assertCompletionListeners(task, numShards+numSkippedShards,
-            numSkippedShards, numShardFailures, false);
-        threadPool.shutdownNow();
-    }
-
-    private static SearchResponse newSearchResponse(int totalShards, int successfulShards, int skippedShards) {
-        InternalSearchResponse response = new InternalSearchResponse(SearchHits.empty(),
-            InternalAggregations.EMPTY, null, null, false, null, 1);
-        return new SearchResponse(response, null, totalShards, successfulShards, skippedShards,
-            100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
-    }
-
-    private void assertCompletionListeners(AsyncSearchTask task,
-                                           int expectedTotalShards,
-                                           int expectedSkippedShards,
-                                           int expectedShardFailures,
-                                           boolean isPartial) throws InterruptedException {
-        int numThreads = randomIntBetween(1, 10);
-        CountDownLatch latch = new CountDownLatch(numThreads);
-        for (int i = 0; i < numThreads; i++) {
-            Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() {
-                @Override
-                public void onResponse(AsyncSearchResponse resp) {
-                    assertThat(resp.getSearchResponse().getTotalShards(), equalTo(expectedTotalShards));
-                    assertThat(resp.getSearchResponse().getSkippedShards(), equalTo(expectedSkippedShards));
-                    assertThat(resp.getSearchResponse().getFailedShards(), equalTo(expectedShardFailures));
-                    assertThat(resp.isPartial(), equalTo(isPartial));
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    throw new AssertionError(e);
-                }
-            }, TimeValue.timeValueMillis(1)));
-            thread.start();
-        }
-        latch.await();
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
deleted file mode 100644
index f71d859f648a3..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-
-import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId;
-
-public class DeleteAsyncSearchRequestTests extends AbstractWireSerializingTestCase<DeleteAsyncSearchAction.Request> {
-    @Override
-    protected Writeable.Reader<DeleteAsyncSearchAction.Request> instanceReader() {
-        return DeleteAsyncSearchAction.Request::new;
-    }
-
-    @Override
-    protected DeleteAsyncSearchAction.Request createTestInstance() {
-        return new DeleteAsyncSearchAction.Request(randomSearchId());
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java
deleted file mode 100644
index 5e3ec4ded0ec8..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-
-public class GetAsyncSearchRequestTests extends AbstractWireSerializingTestCase<GetAsyncSearchAction.Request> {
-    @Override
-    protected Writeable.Reader<GetAsyncSearchAction.Request> instanceReader() {
-        return GetAsyncSearchAction.Request::new;
-    }
-
-    @Override
-    protected GetAsyncSearchAction.Request createTestInstance() {
-        GetAsyncSearchAction.Request req = new GetAsyncSearchAction.Request(randomSearchId());
-        if (randomBoolean()) {
-            req.setWaitForCompletion(TimeValue.timeValueMillis(randomIntBetween(1, 10000)));
-        }
-        if (randomBoolean()) {
-            req.setKeepAlive(TimeValue.timeValueMillis(randomIntBetween(1, 10000)));
-        }
-        return req;
-    }
-
-    static String randomSearchId() {
-        return AsyncSearchId.encode(UUIDs.randomBase64UUID(),
-            new TaskId(randomAlphaOfLengthBetween(10, 20), randomLongBetween(0, Long.MAX_VALUE)));
-    }
-
-    public void testValidateWaitForCompletion() {
-
-    }
-}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
deleted file mode 100644
index 2aae3817205e2..0000000000000
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
-import org.elasticsearch.xpack.core.transform.action.AbstractWireSerializingTransformTestCase;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class SubmitAsyncSearchRequestTests extends AbstractWireSerializingTransformTestCase<SubmitAsyncSearchRequest> {
-    @Override
-    protected Writeable.Reader<SubmitAsyncSearchRequest> instanceReader() {
-        return SubmitAsyncSearchRequest::new;
-    }
-
-    @Override
-    protected SubmitAsyncSearchRequest createTestInstance() {
-        final SubmitAsyncSearchRequest searchRequest;
-        if (randomBoolean()) {
-            searchRequest = new SubmitAsyncSearchRequest(generateRandomStringArray(10, 10, false, false));
-        }  else {
-            searchRequest = new SubmitAsyncSearchRequest();
-        }
-        if (randomBoolean()) {
-            searchRequest.setWaitForCompletion(TimeValue.parseTimeValue(randomPositiveTimeValue(), "wait_for_completion"));
-        }
-        searchRequest.setCleanOnCompletion(randomBoolean());
-        if (randomBoolean()) {
-            searchRequest.setKeepAlive(TimeValue.parseTimeValue(randomPositiveTimeValue(), "keep_alive"));
-        }
-        if (randomBoolean()) {
-            searchRequest.getSearchRequest()
-                .indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
-        }
-        if (randomBoolean()) {
-            searchRequest.getSearchRequest()
-                .preference(randomAlphaOfLengthBetween(3, 10));
-        }
-        if (randomBoolean()) {
-            searchRequest.getSearchRequest().requestCache(randomBoolean());
-        }
-        if (randomBoolean()) {
-            searchRequest.getSearchRequest().searchType(randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH));
-        }
-        if (randomBoolean()) {
-            searchRequest.getSearchRequest().source(randomSearchSourceBuilder());
-        }
-        return searchRequest;
-    }
-
-    protected SearchSourceBuilder randomSearchSourceBuilder() {
-        SearchSourceBuilder source = new SearchSourceBuilder();
-        if (randomBoolean()) {
-            source.query(QueryBuilders.termQuery("foo", "bar"));
-        }
-        if (randomBoolean()) {
-            source.aggregation(AggregationBuilders.max("max").field("field"));
-        }
-        return source;
-    }
-
-    public void testValidateCssMinimizeRoundtrips() {
-        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
-        req.getSearchRequest().setCcsMinimizeRoundtrips(true);
-        ActionRequestValidationException exc = req.validate();
-        assertNotNull(exc);
-        assertThat(exc.validationErrors().size(), equalTo(1));
-        assertThat(exc.validationErrors().get(0), containsString("[ccs_minimize_roundtrips]"));
-    }
-
-    public void testValidateScroll() {
-        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
-        req.getSearchRequest().scroll(TimeValue.timeValueMinutes(5));
-        ActionRequestValidationException exc = req.validate();
-        assertNotNull(exc);
-        assertThat(exc.validationErrors().size(), equalTo(2));
-        // request_cache is activated by default
-        assertThat(exc.validationErrors().get(0), containsString("[request_cache]"));
-        assertThat(exc.validationErrors().get(1), containsString("[scroll]"));
-    }
-
-    public void testValidateKeepAlive() {
-        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
-        req.setKeepAlive(TimeValue.timeValueSeconds(randomIntBetween(1, 59)));
-        ActionRequestValidationException exc = req.validate();
-        assertNotNull(exc);
-        assertThat(exc.validationErrors().size(), equalTo(1));
-        assertThat(exc.validationErrors().get(0), containsString("[keep_alive]"));
-    }
-
-    public void testValidateSuggestOnly() {
-        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
-        req.getSearchRequest().source(new SearchSourceBuilder().suggest(new SuggestBuilder()));
-        ActionRequestValidationException exc = req.validate();
-        assertNotNull(exc);
-        assertThat(exc.validationErrors().size(), equalTo(1));
-        assertThat(exc.validationErrors().get(0), containsString("suggest"));
-    }
-}
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index e2867b1d715a8..85bf1a7500d46 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -105,8 +105,6 @@ testClusters.integTest {
   keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
   user username: "x_pack_rest_user", password: "x-pack-test-password"
-  user username: "cat_test_user", password: "cat-test-password", role: "cat_test_role"
   extraConfigFile nodeKey.name, nodeKey
   extraConfigFile nodeCert.name, nodeCert
-  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
index 28cefdd0a57b4..d1270aeefd203 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
@@ -395,7 +395,7 @@ protected void asyncShardOperation(
 
                         @Override
                         public Executor executor() {
-                            return threadPool.executor(Ccr.CCR_THREAD_POOL_NAME);
+                            return threadPool.executor(ThreadPool.Names.LISTENER);
                         }
 
                         @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
index e523d452c75b1..cbc4e4abc2bfe 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
@@ -132,7 +132,16 @@ protected void innerUpdateMapping(long minRequiredMappingVersion, LongConsumer h
                 final Index followerIndex = params.getFollowShardId().getIndex();
                 final Index leaderIndex = params.getLeaderShardId().getIndex();
                 final Supplier<TimeValue> timeout = () -> isStopped() ? TimeValue.MINUS_ONE : waitForMetadataTimeOut;
-                final ActionListener<IndexMetaData> listener = ActionListener.wrap(
+
+                final Client remoteClient;
+                try {
+                    remoteClient = remoteClient(params);
+                } catch (NoSuchRemoteClusterException e) {
+                    errorHandler.accept(e);
+                    return;
+                }
+
+                CcrRequests.getIndexMetadata(remoteClient, leaderIndex, minRequiredMappingVersion, 0L, timeout, ActionListener.wrap(
                     indexMetaData -> {
                         if (indexMetaData.mapping() == null) {
                             assert indexMetaData.getMappingVersion() == 1;
@@ -146,12 +155,7 @@ protected void innerUpdateMapping(long minRequiredMappingVersion, LongConsumer h
                             errorHandler));
                     },
                     errorHandler
-                );
-                try {
-                    CcrRequests.getIndexMetadata(remoteClient(params), leaderIndex, minRequiredMappingVersion, 0L, timeout, listener);
-                } catch (NoSuchRemoteClusterException e) {
-                    errorHandler.accept(e);
-                }
+                ));
             }
 
             @Override
@@ -420,27 +424,21 @@ protected Scheduler.Cancellable scheduleBackgroundRetentionLeaseRenewal(final Lo
                                         "{} background adding retention lease [{}] while following",
                                         params.getFollowShardId(),
                                         retentionLeaseId);
-                                try {
-                                    final ActionListener<RetentionLeaseActions.Response> wrappedListener = ActionListener.wrap(
-                                        r -> {},
-                                        inner -> {
-                                            /*
-                                             * If this fails that the retention lease already exists, something highly unusual is
-                                             * going on. Log it, and renew again after another renew interval has passed.
-                                             */
-                                            final Throwable innerCause = ExceptionsHelper.unwrapCause(inner);
-                                            logRetentionLeaseFailure(retentionLeaseId, innerCause);
-                                        });
-                                    CcrRetentionLeases.asyncAddRetentionLease(
+                                CcrRetentionLeases.asyncAddRetentionLease(
                                         params.getLeaderShardId(),
                                         retentionLeaseId,
                                         followerGlobalCheckpoint.getAsLong(),
                                         remoteClient(params),
-                                        wrappedListener);
-                                } catch (NoSuchRemoteClusterException rce) {
-                                    // we will attempt to renew again after another renew interval has passed
-                                    logRetentionLeaseFailure(retentionLeaseId, rce);
-                                }
+                                        ActionListener.wrap(
+                                                r -> {},
+                                                inner -> {
+                                                    /*
+                                                     * If this fails that the retention lease already exists, something highly unusual is
+                                                     * going on. Log it, and renew again after another renew interval has passed.
+                                                     */
+                                                    final Throwable innerCause = ExceptionsHelper.unwrapCause(inner);
+                                                    logRetentionLeaseFailure(retentionLeaseId, innerCause);
+                                                }));
                             } else {
                                  // if something else happened, we will attempt to renew again after another renew interval has passed
                             }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index c349ea764101f..7181dd9b6c9a2 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -225,6 +225,7 @@ private NodeConfigurationSource createNodeConfigurationSource(final String leade
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
+        builder.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         // Let cluster state api return quickly in order to speed up auto follow tests:
         builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100));
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
index 9858362c811de..cbfd078452368 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
@@ -48,6 +48,7 @@ protected Settings nodeSettings() {
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
+        builder.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         // Let cluster state api return quickly in order to speed up auto follow tests:
         builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index 4e58023eb260b..1d90ef9488789 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -613,16 +613,6 @@ public boolean isFlattenedAllowed() {
     public boolean isVectorsAllowed() {
         return allowForAllLicenses();
     }
-    
-
-    /**
-     * Determine if Wildcard support should be enabled.
-     * <p>
-     *  Wildcard is available for all license types except {@link OperationMode#MISSING}
-     */
-    public synchronized boolean isWildcardAllowed() {
-        return status.active;
-    }    
 
     public boolean isOdbcAllowed() {
         return isAllowedByLicense(OperationMode.PLATINUM);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 26abe1af66fa0..16a3ce6d5bcb6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -34,9 +34,7 @@
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
+import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage;
@@ -422,11 +420,7 @@ public List<ActionType<? extends ActionResponse>> getClientActions() {
                 DeleteTransformAction.INSTANCE,
                 GetTransformAction.INSTANCE,
                 GetTransformStatsAction.INSTANCE,
-                PreviewTransformAction.INSTANCE,
-                // Async Search
-                SubmitAsyncSearchAction.INSTANCE,
-                GetAsyncSearchAction.INSTANCE,
-                DeleteAsyncSearchAction.INSTANCE
+                PreviewTransformAction.INSTANCE
         );
     }
 
@@ -438,6 +432,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new),
                 // logstash
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.LOGSTASH, LogstashFeatureSetUsage::new),
+                // beats
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.BEATS, BeatsFeatureSetUsage::new),
                 // ML - Custom metadata
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index 740d1015a7eb6..82e9f2553c3ed 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -82,7 +82,11 @@ private XPackSettings() {
 
     /** Setting for enabling or disabling Logstash extensions. Defaults to true. */
     public static final Setting<Boolean> LOGSTASH_ENABLED = Setting.boolSetting("xpack.logstash.enabled", true,
-            Setting.Property.NodeScope, Property.Deprecated);
+            Setting.Property.NodeScope);
+
+    /** Setting for enabling or disabling Beats extensions. Defaults to true. */
+    public static final Setting<Boolean> BEATS_ENABLED = Setting.boolSetting("xpack.beats.enabled", true,
+        Setting.Property.NodeScope);
 
     /**
      * Setting for enabling or disabling the index lifecycle extension. Defaults to true.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java
new file mode 100644
index 0000000000000..1702bf3869df6
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.beats;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+
+public final class BeatsFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    public BeatsFeatureSetUsage(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    public BeatsFeatureSetUsage(boolean available, boolean enabled) {
+        super(XPackField.BEATS, available, enabled);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
index 43cf8b603fdd0..7d621a62cf359 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
@@ -114,8 +114,9 @@ public AutoFollowStats(StreamInput in) throws IOException {
         numberOfFailedFollowIndices = in.readVLong();
         numberOfFailedRemoteClusterStateRequests = in.readVLong();
         numberOfSuccessfulFollowIndices = in.readVLong();
-        recentAutoFollowErrors = new TreeMap<>(
-                in.readMap(StreamInput::readString, in1 -> new Tuple<>(in1.readZLong(), in1.readException())));
+        // note: the casts to the following Writeable.Reader<T> instances are needed by some IDEs (e.g. Eclipse 4.8) as a compiler help
+        recentAutoFollowErrors = new TreeMap<>(in.readMap(StreamInput::readString,
+            in1 -> new Tuple<>(in1.readZLong(), in1.readException())));
         autoFollowedClusters = new TreeMap<>(in.readMap(StreamInput::readString, AutoFollowedCluster::new));
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
index fa5f4c13491d6..3b67e5bdc17d3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
@@ -27,12 +26,7 @@ public FreezeStep(StepKey key, StepKey nextStepKey, Client client) {
     public void performDuringNoSnapshot(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
         getClient().admin().indices().execute(FreezeIndexAction.INSTANCE,
             new FreezeRequest(indexMetaData.getIndex().getName()).masterNodeTimeout(getMasterTimeout(currentState)),
-            ActionListener.wrap(response -> {
-                if (response.isAcknowledged() == false) {
-                    throw new ElasticsearchException("freeze index request failed to be acknowledged");
-                }
-                listener.onResponse(true);
-            }, listener::onFailure));
+            ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java
deleted file mode 100644
index c1839af5a6ec0..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml.action;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-public class EstimateModelMemoryAction extends ActionType<EstimateModelMemoryAction.Response> {
-
-    public static final EstimateModelMemoryAction INSTANCE = new EstimateModelMemoryAction();
-    public static final String NAME = "cluster:admin/xpack/ml/job/estimate_model_memory";
-
-    private EstimateModelMemoryAction() {
-        super(NAME, Response::new);
-    }
-
-    public static class Request extends ActionRequest {
-
-        public static final ParseField ANALYSIS_CONFIG = Job.ANALYSIS_CONFIG;
-        public static final ParseField OVERALL_CARDINALITY = new ParseField("overall_cardinality");
-        public static final ParseField MAX_BUCKET_CARDINALITY = new ParseField("max_bucket_cardinality");
-
-        public static final ObjectParser<Request, Void> PARSER =
-            new ObjectParser<>(NAME, EstimateModelMemoryAction.Request::new);
-
-        static {
-            PARSER.declareObject(Request::setAnalysisConfig, (p, c) -> AnalysisConfig.STRICT_PARSER.apply(p, c).build(), ANALYSIS_CONFIG);
-            PARSER.declareObject(Request::setOverallCardinality,
-                (p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, OVERALL_CARDINALITY)),
-                OVERALL_CARDINALITY);
-            PARSER.declareObject(Request::setMaxBucketCardinality,
-                (p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, MAX_BUCKET_CARDINALITY)),
-                MAX_BUCKET_CARDINALITY);
-        }
-
-        public static Request parseRequest(XContentParser parser) {
-            return PARSER.apply(parser, null);
-        }
-
-        private AnalysisConfig analysisConfig;
-        private Map<String, Long> overallCardinality = Collections.emptyMap();
-        private Map<String, Long> maxBucketCardinality = Collections.emptyMap();
-
-        public Request() {
-            super();
-        }
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            this.analysisConfig = in.readBoolean() ? new AnalysisConfig(in) : null;
-            this.overallCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
-            this.maxBucketCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            if (analysisConfig != null) {
-                out.writeBoolean(true);
-                analysisConfig.writeTo(out);
-            } else {
-                out.writeBoolean(false);
-            }
-            out.writeMap(overallCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
-            out.writeMap(maxBucketCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            if (analysisConfig == null) {
-                ActionRequestValidationException e = new ActionRequestValidationException();
-                e.addValidationError("[" + ANALYSIS_CONFIG.getPreferredName() + "] was not specified");
-                return e;
-            }
-            return null;
-        }
-
-        public AnalysisConfig getAnalysisConfig() {
-            return analysisConfig;
-        }
-
-        public void setAnalysisConfig(AnalysisConfig analysisConfig) {
-            this.analysisConfig = ExceptionsHelper.requireNonNull(analysisConfig, ANALYSIS_CONFIG);
-        }
-
-        public Map<String, Long> getOverallCardinality() {
-            return overallCardinality;
-        }
-
-        public void setOverallCardinality(Map<String, Long> overallCardinality) {
-            this.overallCardinality =
-                Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(overallCardinality, OVERALL_CARDINALITY));
-        }
-
-        public Map<String, Long> getMaxBucketCardinality() {
-            return maxBucketCardinality;
-        }
-
-        public void setMaxBucketCardinality(Map<String, Long> maxBucketCardinality) {
-            this.maxBucketCardinality =
-                Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(maxBucketCardinality, MAX_BUCKET_CARDINALITY));
-        }
-
-        private static long parseNonNegativeLong(XContentParser parser, ParseField enclosingField) throws IOException {
-            long value = parser.longValue();
-            if (value < 0) {
-                throw ExceptionsHelper.badRequestException("[{}] contained negative cardinality [{}]",
-                    enclosingField.getPreferredName(), value);
-            }
-            return value;
-        }
-    }
-
-    public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
-
-        public RequestBuilder(ElasticsearchClient client, EstimateModelMemoryAction action) {
-            super(client, action, new Request());
-        }
-    }
-
-    public static class Response extends ActionResponse implements ToXContentObject {
-
-        private static final ParseField MODEL_MEMORY_ESTIMATE = new ParseField("model_memory_estimate");
-
-        private final ByteSizeValue modelMemoryEstimate;
-
-        public Response(ByteSizeValue modelMemoryEstimate) {
-            this.modelMemoryEstimate = Objects.requireNonNull(modelMemoryEstimate);
-        }
-
-        public Response(StreamInput in) throws IOException {
-            modelMemoryEstimate = new ByteSizeValue(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            modelMemoryEstimate.writeTo(out);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(MODEL_MEMORY_ESTIMATE.getPreferredName(), modelMemoryEstimate.getStringRep());
-            builder.endObject();
-            return builder;
-        }
-
-        public ByteSizeValue getModelMemoryEstimate() {
-            return modelMemoryEstimate;
-        }
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java
index 664162bb80cb4..04ced8a2f4c06 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java
@@ -33,7 +33,7 @@ public class BoostedTreeParams implements ToXContentFragment, Writeable {
     public static final ParseField LAMBDA = new ParseField("lambda");
     public static final ParseField GAMMA = new ParseField("gamma");
     public static final ParseField ETA = new ParseField("eta");
-    public static final ParseField MAX_TREES = new ParseField("max_trees", "maximum_number_trees");
+    public static final ParseField MAXIMUM_NUMBER_TREES = new ParseField("maximum_number_trees");
     public static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
     public static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
 
@@ -41,7 +41,7 @@ static void declareFields(AbstractObjectParser<?, Void> parser) {
         parser.declareDouble(optionalConstructorArg(), LAMBDA);
         parser.declareDouble(optionalConstructorArg(), GAMMA);
         parser.declareDouble(optionalConstructorArg(), ETA);
-        parser.declareInt(optionalConstructorArg(), MAX_TREES);
+        parser.declareInt(optionalConstructorArg(), MAXIMUM_NUMBER_TREES);
         parser.declareDouble(optionalConstructorArg(), FEATURE_BAG_FRACTION);
         parser.declareInt(optionalConstructorArg(), NUM_TOP_FEATURE_IMPORTANCE_VALUES);
     }
@@ -49,14 +49,14 @@ static void declareFields(AbstractObjectParser<?, Void> parser) {
     private final Double lambda;
     private final Double gamma;
     private final Double eta;
-    private final Integer maxTrees;
+    private final Integer maximumNumberTrees;
     private final Double featureBagFraction;
     private final Integer numTopFeatureImportanceValues;
 
     public BoostedTreeParams(@Nullable Double lambda,
                              @Nullable Double gamma,
                              @Nullable Double eta,
-                             @Nullable Integer maxTrees,
+                             @Nullable Integer maximumNumberTrees,
                              @Nullable Double featureBagFraction,
                              @Nullable Integer numTopFeatureImportanceValues) {
         if (lambda != null && lambda < 0) {
@@ -68,8 +68,8 @@ public BoostedTreeParams(@Nullable Double lambda,
         if (eta != null && (eta < 0.001 || eta > 1)) {
             throw ExceptionsHelper.badRequestException("[{}] must be a double in [0.001, 1]", ETA.getPreferredName());
         }
-        if (maxTrees != null && (maxTrees <= 0 || maxTrees > 2000)) {
-            throw ExceptionsHelper.badRequestException("[{}] must be an integer in [1, 2000]", MAX_TREES.getPreferredName());
+        if (maximumNumberTrees != null && (maximumNumberTrees <= 0 || maximumNumberTrees > 2000)) {
+            throw ExceptionsHelper.badRequestException("[{}] must be an integer in [1, 2000]", MAXIMUM_NUMBER_TREES.getPreferredName());
         }
         if (featureBagFraction != null && (featureBagFraction <= 0 || featureBagFraction > 1.0)) {
             throw ExceptionsHelper.badRequestException("[{}] must be a double in (0, 1]", FEATURE_BAG_FRACTION.getPreferredName());
@@ -81,7 +81,7 @@ public BoostedTreeParams(@Nullable Double lambda,
         this.lambda = lambda;
         this.gamma = gamma;
         this.eta = eta;
-        this.maxTrees = maxTrees;
+        this.maximumNumberTrees = maximumNumberTrees;
         this.featureBagFraction = featureBagFraction;
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
     }
@@ -90,7 +90,7 @@ public BoostedTreeParams(@Nullable Double lambda,
         lambda = in.readOptionalDouble();
         gamma = in.readOptionalDouble();
         eta = in.readOptionalDouble();
-        maxTrees = in.readOptionalVInt();
+        maximumNumberTrees = in.readOptionalVInt();
         featureBagFraction = in.readOptionalDouble();
         if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
             numTopFeatureImportanceValues = in.readOptionalInt();
@@ -104,7 +104,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalDouble(lambda);
         out.writeOptionalDouble(gamma);
         out.writeOptionalDouble(eta);
-        out.writeOptionalVInt(maxTrees);
+        out.writeOptionalVInt(maximumNumberTrees);
         out.writeOptionalDouble(featureBagFraction);
         if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
             out.writeOptionalInt(numTopFeatureImportanceValues);
@@ -122,8 +122,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (eta != null) {
             builder.field(ETA.getPreferredName(), eta);
         }
-        if (maxTrees != null) {
-            builder.field(MAX_TREES.getPreferredName(), maxTrees);
+        if (maximumNumberTrees != null) {
+            builder.field(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
         }
         if (featureBagFraction != null) {
             builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -145,8 +145,8 @@ Map<String, Object> getParams() {
         if (eta != null) {
             params.put(ETA.getPreferredName(), eta);
         }
-        if (maxTrees != null) {
-            params.put(MAX_TREES.getPreferredName(), maxTrees);
+        if (maximumNumberTrees != null) {
+            params.put(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
         }
         if (featureBagFraction != null) {
             params.put(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -165,14 +165,14 @@ public boolean equals(Object o) {
         return Objects.equals(lambda, that.lambda)
             && Objects.equals(gamma, that.gamma)
             && Objects.equals(eta, that.eta)
-            && Objects.equals(maxTrees, that.maxTrees)
+            && Objects.equals(maximumNumberTrees, that.maximumNumberTrees)
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(numTopFeatureImportanceValues, that.numTopFeatureImportanceValues);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues);
+        return Objects.hash(lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues);
     }
 
     public static Builder builder() {
@@ -184,7 +184,7 @@ public static class Builder {
         private Double lambda;
         private Double gamma;
         private Double eta;
-        private Integer maxTrees;
+        private Integer maximumNumberTrees;
         private Double featureBagFraction;
         private Integer numTopFeatureImportanceValues;
 
@@ -194,7 +194,7 @@ private Builder() {}
             this.lambda = params.lambda;
             this.gamma = params.gamma;
             this.eta = params.eta;
-            this.maxTrees = params.maxTrees;
+            this.maximumNumberTrees = params.maximumNumberTrees;
             this.featureBagFraction = params.featureBagFraction;
             this.numTopFeatureImportanceValues = params.numTopFeatureImportanceValues;
         }
@@ -214,8 +214,8 @@ public Builder setEta(Double eta) {
             return this;
         }
 
-        public Builder setMaxTrees(Integer maxTrees) {
-            this.maxTrees = maxTrees;
+        public Builder setMaximumNumberTrees(Integer maximumNumberTrees) {
+            this.maximumNumberTrees = maximumNumberTrees;
             return this;
         }
 
@@ -230,7 +230,7 @@ public Builder setNumTopFeatureImportanceValues(Integer numTopFeatureImportanceV
         }
 
         public BoostedTreeParams build() {
-            return new BoostedTreeParams(lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues);
+            return new BoostedTreeParams(lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues);
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
index c0965f8d7ebb5..d9289a5bcf1ac 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.FieldAliasMapper;
@@ -22,7 +21,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -39,23 +37,15 @@ public class Classification implements DataFrameAnalysis {
 
     public static final ParseField DEPENDENT_VARIABLE = new ParseField("dependent_variable");
     public static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
-    public static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
     public static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
     public static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
     public static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
     private static final String STATE_DOC_ID_SUFFIX = "_classification_state#1";
 
-    private static final String NUM_CLASSES = "num_classes";
-
     private static final ConstructingObjectParser<Classification, Void> LENIENT_PARSER = createParser(true);
     private static final ConstructingObjectParser<Classification, Void> STRICT_PARSER = createParser(false);
 
-    /**
-     * The max number of classes classification supports
-     */
-    private static final int MAX_DEPENDENT_VARIABLE_CARDINALITY = 30;
-
     private static ConstructingObjectParser<Classification, Void> createParser(boolean lenient) {
         ConstructingObjectParser<Classification, Void> parser = new ConstructingObjectParser<>(
             NAME.getPreferredName(),
@@ -64,19 +54,12 @@ private static ConstructingObjectParser<Classification, Void> createParser(boole
                 (String) a[0],
                 new BoostedTreeParams((Double) a[1], (Double) a[2], (Double) a[3], (Integer) a[4], (Double) a[5], (Integer) a[6]),
                 (String) a[7],
-                (ClassAssignmentObjective) a[8],
-                (Integer) a[9],
-                (Double) a[10],
-                (Long) a[11]));
+                (Integer) a[8],
+                (Double) a[9],
+                (Long) a[10]));
         parser.declareString(constructorArg(), DEPENDENT_VARIABLE);
         BoostedTreeParams.declareFields(parser);
         parser.declareString(optionalConstructorArg(), PREDICTION_FIELD_NAME);
-        parser.declareField(optionalConstructorArg(), p -> {
-            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                return ClassAssignmentObjective.fromString(p.text());
-            }
-            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-        }, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
         parser.declareInt(optionalConstructorArg(), NUM_TOP_CLASSES);
         parser.declareDouble(optionalConstructorArg(), TRAINING_PERCENT);
         parser.declareLong(optionalConstructorArg(), RANDOMIZE_SEED);
@@ -106,7 +89,6 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
     private final String dependentVariable;
     private final BoostedTreeParams boostedTreeParams;
     private final String predictionFieldName;
-    private final ClassAssignmentObjective classAssignmentObjective;
     private final int numTopClasses;
     private final double trainingPercent;
     private final long randomizeSeed;
@@ -114,7 +96,6 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
     public Classification(String dependentVariable,
                           BoostedTreeParams boostedTreeParams,
                           @Nullable String predictionFieldName,
-                          @Nullable ClassAssignmentObjective classAssignmentObjective,
                           @Nullable Integer numTopClasses,
                           @Nullable Double trainingPercent,
                           @Nullable Long randomizeSeed) {
@@ -127,26 +108,19 @@ public Classification(String dependentVariable,
         this.dependentVariable = ExceptionsHelper.requireNonNull(dependentVariable, DEPENDENT_VARIABLE);
         this.boostedTreeParams = ExceptionsHelper.requireNonNull(boostedTreeParams, BoostedTreeParams.NAME);
         this.predictionFieldName = predictionFieldName == null ? dependentVariable + "_prediction" : predictionFieldName;
-        this.classAssignmentObjective = classAssignmentObjective == null ?
-            ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL : classAssignmentObjective;
         this.numTopClasses = numTopClasses == null ? DEFAULT_NUM_TOP_CLASSES : numTopClasses;
         this.trainingPercent = trainingPercent == null ? 100.0 : trainingPercent;
         this.randomizeSeed = randomizeSeed == null ? Randomness.get().nextLong() : randomizeSeed;
     }
 
     public Classification(String dependentVariable) {
-        this(dependentVariable, BoostedTreeParams.builder().build(), null, null, null, null, null);
+        this(dependentVariable, BoostedTreeParams.builder().build(), null, null, null, null);
     }
 
     public Classification(StreamInput in) throws IOException {
         dependentVariable = in.readString();
         boostedTreeParams = new BoostedTreeParams(in);
         predictionFieldName = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
-            classAssignmentObjective = in.readEnum(ClassAssignmentObjective.class);
-        } else {
-            classAssignmentObjective = ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL;
-        }
         numTopClasses = in.readOptionalVInt();
         trainingPercent = in.readDouble();
         if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
@@ -168,10 +142,6 @@ public String getPredictionFieldName() {
         return predictionFieldName;
     }
 
-    public ClassAssignmentObjective getClassAssignmentObjective() {
-        return classAssignmentObjective;
-    }
-
     public int getNumTopClasses() {
         return numTopClasses;
     }
@@ -194,9 +164,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(dependentVariable);
         boostedTreeParams.writeTo(out);
         out.writeOptionalString(predictionFieldName);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
-            out.writeEnum(classAssignmentObjective);
-        }
         out.writeOptionalVInt(numTopClasses);
         out.writeDouble(trainingPercent);
         if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
@@ -211,7 +178,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         boostedTreeParams.toXContent(builder, params);
-        builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
         builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         if (predictionFieldName != null) {
             builder.field(PREDICTION_FIELD_NAME.getPreferredName(), predictionFieldName);
@@ -225,20 +191,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Map<String, Object> getParams(FieldInfo fieldInfo) {
+    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
         Map<String, Object> params = new HashMap<>();
         params.put(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         params.putAll(boostedTreeParams.getParams());
-        params.put(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
         params.put(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         if (predictionFieldName != null) {
             params.put(PREDICTION_FIELD_NAME.getPreferredName(), predictionFieldName);
         }
-        String predictionFieldType = getPredictionFieldType(fieldInfo.getTypes(dependentVariable));
+        String predictionFieldType = getPredictionFieldType(extractedFields.get(dependentVariable));
         if (predictionFieldType != null) {
             params.put(PREDICTION_FIELD_TYPE, predictionFieldType);
         }
-        params.put(NUM_CLASSES, fieldInfo.getCardinality(dependentVariable));
         return params;
     }
 
@@ -280,7 +244,7 @@ public List<RequiredField> getRequiredFields() {
     @Override
     public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
         // This restriction is due to the fact that currently the C++ backend only supports binomial classification.
-        return Collections.singletonList(FieldCardinalityConstraint.between(dependentVariable, 2, MAX_DEPENDENT_VARIABLE_CARDINALITY));
+        return Collections.singletonList(FieldCardinalityConstraint.between(dependentVariable, 2, 2));
     }
 
     @SuppressWarnings("unchecked")
@@ -339,7 +303,6 @@ public boolean equals(Object o) {
         return Objects.equals(dependentVariable, that.dependentVariable)
             && Objects.equals(boostedTreeParams, that.boostedTreeParams)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
-            && Objects.equals(classAssignmentObjective, that.classAssignmentObjective)
             && Objects.equals(numTopClasses, that.numTopClasses)
             && trainingPercent == that.trainingPercent
             && randomizeSeed == that.randomizeSeed;
@@ -347,20 +310,6 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(dependentVariable, boostedTreeParams, predictionFieldName, classAssignmentObjective,
-                            numTopClasses, trainingPercent, randomizeSeed);
-    }
-
-    public enum ClassAssignmentObjective {
-        MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
-
-        public static ClassAssignmentObjective fromString(String value) {
-            return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
-        }
-
-        @Override
-        public String toString() {
-            return name().toLowerCase(Locale.ROOT);
-        }
+        return Objects.hash(dependentVariable, boostedTreeParams, predictionFieldName, numTopClasses, trainingPercent, randomizeSeed);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
index 941224dc30a67..664b38e4fc05d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.ml.dataframe.analyses;
 
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 
@@ -17,9 +16,9 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
 
     /**
      * @return The analysis parameters as a map
-     * @param fieldInfo Information about the fields like types and cardinalities
+     * @param extractedFields map of (name, types) for all the extracted fields
      */
-    Map<String, Object> getParams(FieldInfo fieldInfo);
+    Map<String, Object> getParams(Map<String, Set<String>> extractedFields);
 
     /**
      * @return {@code true} if this analysis supports fields with categorical values (i.e. text, keyword, ip)
@@ -65,27 +64,4 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
      * Returns the document id for the analysis state
      */
     String getStateDocId(String jobId);
-
-    /**
-     * Summarizes information about the fields that is necessary for analysis to generate
-     * the parameters needed for the process configuration.
-     */
-    interface FieldInfo {
-
-        /**
-         * Returns the types for the given field or {@code null} if the field is unknown
-         * @param field the field whose types to return
-         * @return the types for the given field or {@code null} if the field is unknown
-         */
-        @Nullable
-        Set<String> getTypes(String field);
-
-        /**
-         * Returns the cardinality of the given field or {@code null} if there is no cardinality for that field
-         * @param field the field whose cardinality to get
-         * @return the cardinality of the given field or {@code null} if there is no cardinality for that field
-         */
-        @Nullable
-        Long getCardinality(String field);
-    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
index 2c83afa87808d..654d5ba4d1a29 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
@@ -192,7 +192,7 @@ public int hashCode() {
     }
 
     @Override
-    public Map<String, Object> getParams(FieldInfo fieldInfo) {
+    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
         Map<String, Object> params = new HashMap<>();
         if (nNeighbors != null) {
             params.put(N_NEIGHBORS.getPreferredName(), nNeighbors);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
index d8c490ddcb804..86f8039090c74 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
@@ -155,7 +155,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Map<String, Object> getParams(FieldInfo fieldInfo) {
+    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
         Map<String, Object> params = new HashMap<>();
         params.put(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         params.putAll(boostedTreeParams.getParams());
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
index 9fdba68d4cda7..1a79dff41e10c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
@@ -66,7 +66,7 @@ default <T extends EvaluationMetric> List<T> initMetrics(@Nullable List<T> parse
      * Builds the search required to collect data to compute the evaluation result
      * @param userProvidedQueryBuilder User-provided query that must be respected when collecting data
      */
-    default SearchSourceBuilder buildSearch(EvaluationParameters parameters, QueryBuilder userProvidedQueryBuilder) {
+    default SearchSourceBuilder buildSearch(QueryBuilder userProvidedQueryBuilder) {
         Objects.requireNonNull(userProvidedQueryBuilder);
         BoolQueryBuilder boolQuery =
             QueryBuilders.boolQuery()
@@ -78,8 +78,7 @@ default SearchSourceBuilder buildSearch(EvaluationParameters parameters, QueryBu
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).query(boolQuery);
         for (EvaluationMetric metric : getMetrics()) {
             // Fetch aggregations requested by individual metrics
-            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs =
-                metric.aggs(parameters, getActualField(), getPredictedField());
+            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs = metric.aggs(getActualField(), getPredictedField());
             aggs.v1().forEach(searchSourceBuilder::aggregation);
             aggs.v2().forEach(searchSourceBuilder::aggregation);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java
index a5c3d657f55e9..8a106175ace91 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java
@@ -28,14 +28,11 @@ public interface EvaluationMetric extends ToXContentObject, NamedWriteable {
 
     /**
      * Builds the aggregation that collect required data to compute the metric
-     * @param parameters settings that may be needed by aggregations
      * @param actualField the field that stores the actual value
      * @param predictedField the field that stores the predicted value (class name or probability)
      * @return the aggregations required to compute the metric
      */
-    Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                           String actualField,
-                                                                           String predictedField);
+    Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField);
 
     /**
      * Processes given aggregations as a step towards computing result
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java
deleted file mode 100644
index e834efc7e67f7..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml.dataframe.evaluation;
-
-/**
- * Encapsulates parameters needed by evaluation.
- */
-public class EvaluationParameters {
-
-    /**
-     * Maximum number of buckets allowed in any single search request.
-     */
-    private final int maxBuckets;
-
-    public EvaluationParameters(int maxBuckets) {
-        this.maxBuckets = maxBuckets;
-    }
-
-    public int getMaxBuckets() {
-        return maxBuckets;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
index 440a46540cecb..c6636329a65d9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
@@ -24,13 +24,14 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -64,6 +65,12 @@ public class Accuracy implements EvaluationMetric {
 
     static final String OVERALL_ACCURACY_AGG_NAME = "classification_overall_accuracy";
 
+    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
+
+    private static Script buildScript(Object...args) {
+        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
+    }
+
     private static final ObjectParser<Accuracy, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Accuracy::new);
 
     public static Accuracy fromXContent(XContentParser parser) {
@@ -96,20 +103,16 @@ public String getName() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                        String actualField,
-                                                                                        String predictedField) {
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
         // Store given {@code actualField} for the purpose of generating error message in {@code process}.
         this.actualField.trySet(actualField);
         List<AggregationBuilder> aggs = new ArrayList<>();
         List<PipelineAggregationBuilder> pipelineAggs = new ArrayList<>();
         if (overallAccuracy.get() == null) {
-            Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
-            aggs.add(AggregationBuilders.avg(OVERALL_ACCURACY_AGG_NAME).script(script));
+            aggs.add(AggregationBuilders.avg(OVERALL_ACCURACY_AGG_NAME).script(buildScript(actualField, predictedField)));
         }
         if (result.get() == null) {
-            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> matrixAggs =
-                matrix.aggs(parameters, actualField, predictedField);
+            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> matrixAggs = matrix.aggs(actualField, predictedField);
             aggs.addAll(matrixAggs.v1());
             pipelineAggs.addAll(matrixAggs.v2());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java
index f041e66698aa6..e2762bd7d7986 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.search.aggregations.metrics.Cardinality;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -73,9 +72,9 @@ public static MulticlassConfusionMatrix fromXContent(XContentParser parser) {
     }
 
     static final String STEP_1_AGGREGATE_BY_ACTUAL_CLASS = NAME.getPreferredName() + "_step_1_by_actual_class";
-    static final String STEP_1_CARDINALITY_OF_ACTUAL_CLASS = NAME.getPreferredName() + "_step_1_cardinality_of_actual_class";
     static final String STEP_2_AGGREGATE_BY_ACTUAL_CLASS = NAME.getPreferredName() + "_step_2_by_actual_class";
     static final String STEP_2_AGGREGATE_BY_PREDICTED_CLASS = NAME.getPreferredName() + "_step_2_by_predicted_class";
+    static final String STEP_2_CARDINALITY_OF_ACTUAL_CLASS = NAME.getPreferredName() + "_step_2_cardinality_of_actual_class";
     private static final String OTHER_BUCKET_KEY = "_other_";
     private static final String DEFAULT_AGG_NAME_PREFIX = "";
     private static final int DEFAULT_SIZE = 10;
@@ -84,9 +83,6 @@ public static MulticlassConfusionMatrix fromXContent(XContentParser parser) {
     private final int size;
     private final String aggNamePrefix;
     private final SetOnce<List<String>> topActualClassNames = new SetOnce<>();
-    private final SetOnce<Long> actualClassesCardinality = new SetOnce<>();
-    /** Accumulates actual classes processed so far. It may take more than 1 call to #process method to fill this field completely. */
-    private final List<ActualClass> actualClasses = new ArrayList<>();
     private final SetOnce<Result> result = new SetOnce<>();
 
     public MulticlassConfusionMatrix() {
@@ -125,45 +121,34 @@ public int getSize() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                        String actualField,
-                                                                                        String predictedField) {
-        if (topActualClassNames.get() == null && actualClassesCardinality.get() == null) {  // This is step 1
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
+        if (topActualClassNames.get() == null) {  // This is step 1
             return Tuple.tuple(
                 List.of(
                     AggregationBuilders.terms(aggName(STEP_1_AGGREGATE_BY_ACTUAL_CLASS))
                         .field(actualField)
                         .order(List.of(BucketOrder.count(false), BucketOrder.key(true)))
-                        .size(size),
-                    AggregationBuilders.cardinality(aggName(STEP_1_CARDINALITY_OF_ACTUAL_CLASS))
-                        .field(actualField)),
+                        .size(size)),
                 List.of());
         }
-        if (result.get() == null) {  // These are steps 2, 3, 4 etc.
-            KeyedFilter[] keyedFiltersPredicted =
-                topActualClassNames.get().stream()
-                    .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(predictedField, className)))
-                    .toArray(KeyedFilter[]::new);
-            // Knowing exactly how many buckets does each aggregation use, we can choose the size of the batch so that
-            // too_many_buckets_exception exception is not thrown.
-            // The only exception is when "search.max_buckets" is set far too low to even have 1 actual class in the batch.
-            // In such case, the exception will be thrown telling the user they should increase the value of "search.max_buckets".
-            int actualClassesPerBatch = Math.max(parameters.getMaxBuckets() / (topActualClassNames.get().size() + 2), 1);
+        if (result.get() == null) {  // This is step 2
             KeyedFilter[] keyedFiltersActual =
                 topActualClassNames.get().stream()
-                    .skip(actualClasses.size())
-                    .limit(actualClassesPerBatch)
                     .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(actualField, className)))
                     .toArray(KeyedFilter[]::new);
-            if (keyedFiltersActual.length > 0) {
-                return Tuple.tuple(
-                    List.of(
-                        AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS), keyedFiltersActual)
-                            .subAggregation(AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_PREDICTED_CLASS), keyedFiltersPredicted)
-                                .otherBucket(true)
-                                .otherBucketKey(OTHER_BUCKET_KEY))),
-                    List.of());
-            }
+            KeyedFilter[] keyedFiltersPredicted =
+                topActualClassNames.get().stream()
+                    .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(predictedField, className)))
+                    .toArray(KeyedFilter[]::new);
+            return Tuple.tuple(
+                List.of(
+                    AggregationBuilders.cardinality(aggName(STEP_2_CARDINALITY_OF_ACTUAL_CLASS))
+                        .field(actualField),
+                    AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS), keyedFiltersActual)
+                        .subAggregation(AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_PREDICTED_CLASS), keyedFiltersPredicted)
+                            .otherBucket(true)
+                            .otherBucketKey(OTHER_BUCKET_KEY))),
+                List.of());
         }
         return Tuple.tuple(List.of(), List.of());
     }
@@ -174,12 +159,10 @@ public void process(Aggregations aggs) {
             Terms termsAgg = aggs.get(aggName(STEP_1_AGGREGATE_BY_ACTUAL_CLASS));
             topActualClassNames.set(termsAgg.getBuckets().stream().map(Terms.Bucket::getKeyAsString).sorted().collect(Collectors.toList()));
         }
-        if (actualClassesCardinality.get() == null && aggs.get(aggName(STEP_1_CARDINALITY_OF_ACTUAL_CLASS)) != null) {
-            Cardinality cardinalityAgg = aggs.get(aggName(STEP_1_CARDINALITY_OF_ACTUAL_CLASS));
-            actualClassesCardinality.set(cardinalityAgg.getValue());
-        }
         if (result.get() == null && aggs.get(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS)) != null) {
+            Cardinality cardinalityAgg = aggs.get(aggName(STEP_2_CARDINALITY_OF_ACTUAL_CLASS));
             Filters filtersAgg = aggs.get(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS));
+            List<ActualClass> actualClasses = new ArrayList<>(filtersAgg.getBuckets().size());
             for (Filters.Bucket bucket : filtersAgg.getBuckets()) {
                 String actualClass = bucket.getKeyAsString();
                 long actualClassDocCount = bucket.getDocCount();
@@ -198,9 +181,7 @@ public void process(Aggregations aggs) {
                 predictedClasses.sort(comparing(PredictedClass::getPredictedClass));
                 actualClasses.add(new ActualClass(actualClass, actualClassDocCount, predictedClasses, otherPredictedClassDocCount));
             }
-            if (actualClasses.size() == topActualClassNames.get().size()) {
-                result.set(new Result(actualClasses, Math.max(actualClassesCardinality.get() - size, 0)));
-            }
+            result.set(new Result(actualClasses, Math.max(cardinalityAgg.getValue() - size, 0)));
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
deleted file mode 100644
index 152bef11c09a2..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification;
-
-import org.elasticsearch.script.Script;
-
-import java.text.MessageFormat;
-import java.util.Locale;
-
-/**
- * Painless scripts used by classification metrics in this package.
- */
-final class PainlessScripts {
-
-    /**
-     * Template for the comparison script.
-     * It uses "String.valueOf" method in case the mapping types of the two fields are different.
-     */
-    private static final MessageFormat COMPARISON_SCRIPT_TEMPLATE =
-        new MessageFormat("String.valueOf(doc[''{0}''].value).equals(String.valueOf(doc[''{1}''].value))", Locale.ROOT);
-
-    /**
-     * Builds script that tests field values equality for the given actual and predicted field names.
-     *
-     * @param actualField name of the actual field
-     * @param predictedField name of the predicted field
-     * @return script that tests whether the values of actualField and predictedField are equal
-     */
-    static Script buildIsEqualScript(String actualField, String predictedField) {
-        return new Script(COMPARISON_SCRIPT_TEMPLATE.format(new Object[]{ actualField, predictedField }));
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
index c2fe3e4069fcc..73c7723c86b96 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
@@ -30,13 +30,14 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -57,12 +58,17 @@ public class Precision implements EvaluationMetric {
 
     public static final ParseField NAME = new ParseField("precision");
 
+    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
     private static final String AGG_NAME_PREFIX = "classification_precision_";
     static final String ACTUAL_CLASSES_NAMES_AGG_NAME = AGG_NAME_PREFIX + "by_actual_class";
     static final String BY_PREDICTED_CLASS_AGG_NAME = AGG_NAME_PREFIX + "by_predicted_class";
     static final String PER_PREDICTED_CLASS_PRECISION_AGG_NAME = AGG_NAME_PREFIX + "per_predicted_class_precision";
     static final String AVG_PRECISION_AGG_NAME = AGG_NAME_PREFIX + "avg_precision";
 
+    private static Script buildScript(Object...args) {
+        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
+    }
+
     private static final ObjectParser<Precision, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Precision::new);
 
     public static Precision fromXContent(XContentParser parser) {
@@ -90,9 +96,7 @@ public String getName() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                        String actualField,
-                                                                                        String predictedField) {
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
         // Store given {@code actualField} for the purpose of generating error message in {@code process}.
         this.actualField.trySet(actualField);
         if (topActualClassNames.get() == null) {  // This is step 1
@@ -109,7 +113,7 @@ public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> a
                 topActualClassNames.get().stream()
                     .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(predictedField, className)))
                     .toArray(KeyedFilter[]::new);
-            Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
+            Script script = buildScript(actualField, predictedField);
             return Tuple.tuple(
                 List.of(
                     AggregationBuilders.filters(BY_PREDICTED_CLASS_AGG_NAME, keyedFiltersPredicted)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
index 2fdd979fcb825..0358820cc509c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
@@ -20,20 +20,20 @@
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -53,11 +53,16 @@ public class Recall implements EvaluationMetric {
 
     public static final ParseField NAME = new ParseField("recall");
 
+    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
     private static final String AGG_NAME_PREFIX = "classification_recall_";
     static final String BY_ACTUAL_CLASS_AGG_NAME = AGG_NAME_PREFIX + "by_actual_class";
     static final String PER_ACTUAL_CLASS_RECALL_AGG_NAME = AGG_NAME_PREFIX + "per_actual_class_recall";
     static final String AVG_RECALL_AGG_NAME = AGG_NAME_PREFIX + "avg_recall";
 
+    private static Script buildScript(Object...args) {
+        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
+    }
+
     private static final ObjectParser<Recall, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Recall::new);
 
     public static Recall fromXContent(XContentParser parser) {
@@ -84,20 +89,17 @@ public String getName() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                        String actualField,
-                                                                                        String predictedField) {
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
         // Store given {@code actualField} for the purpose of generating error message in {@code process}.
         this.actualField.trySet(actualField);
         if (result.get() != null) {
             return Tuple.tuple(List.of(), List.of());
         }
-        Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
+        Script script = buildScript(actualField, predictedField);
         return Tuple.tuple(
             List.of(
                 AggregationBuilders.terms(BY_ACTUAL_CLASS_AGG_NAME)
                     .field(actualField)
-                    .order(List.of(BucketOrder.count(false), BucketOrder.key(true)))
                     .size(MAX_CLASSES_CARDINALITY)
                     .subAggregation(AggregationBuilders.avg(PER_ACTUAL_CLASS_RECALL_AGG_NAME).script(script))),
             List.of(
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java
index 982c68dfb58af..15a0522d72432 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.text.MessageFormat;
@@ -66,9 +65,7 @@ public String getName() {
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                  String actualField,
-                                                                                  String predictedField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java
index ac142631ab968..d3a04c415c047 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.text.MessageFormat;
@@ -71,9 +70,7 @@ public String getName() {
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                  String actualField,
-                                                                                  String predictedField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
index fdd7cd2ce2d6c..e09e03b1208aa 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -66,9 +65,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                  String actualField,
-                                                                                  String predictedProbabilityField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedProbabilityField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
index db0f9b95d1380..3f62a0b144eec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.search.aggregations.metrics.Percentiles;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -128,9 +127,7 @@ public int hashCode() {
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                  String actualField,
-                                                                                  String predictedProbabilityField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedProbabilityField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
index e7cb624016dd8..33e19cb2882d2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
@@ -31,7 +31,6 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -61,7 +60,6 @@ public class TrainedModelConfig implements ToXContentObject, Writeable {
     public static final ParseField ESTIMATED_HEAP_MEMORY_USAGE_BYTES = new ParseField("estimated_heap_memory_usage_bytes");
     public static final ParseField ESTIMATED_OPERATIONS = new ParseField("estimated_operations");
     public static final ParseField LICENSE_LEVEL = new ParseField("license_level");
-    public static final ParseField DEFAULT_FIELD_MAP = new ParseField("default_field_map");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
     public static final ObjectParser<TrainedModelConfig.Builder, Void> LENIENT_PARSER = createParser(true);
@@ -92,7 +90,6 @@ private static ObjectParser<TrainedModelConfig.Builder, Void> createParser(boole
             DEFINITION);
         parser.declareString(TrainedModelConfig.Builder::setLazyDefinition, COMPRESSED_DEFINITION);
         parser.declareString(TrainedModelConfig.Builder::setLicenseLevel, LICENSE_LEVEL);
-        parser.declareObject(TrainedModelConfig.Builder::setDefaultFieldMap, (p, c) -> p.mapStrings(), DEFAULT_FIELD_MAP);
         return parser;
     }
 
@@ -111,7 +108,6 @@ public static TrainedModelConfig.Builder fromXContent(XContentParser parser, boo
     private final long estimatedHeapMemory;
     private final long estimatedOperations;
     private final License.OperationMode licenseLevel;
-    private final Map<String, String> defaultFieldMap;
 
     private final LazyModelDefinition definition;
 
@@ -126,8 +122,7 @@ public static TrainedModelConfig.Builder fromXContent(XContentParser parser, boo
                        TrainedModelInput input,
                        Long estimatedHeapMemory,
                        Long estimatedOperations,
-                       String licenseLevel,
-                       Map<String, String> defaultFieldMap) {
+                       String licenseLevel) {
         this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
         this.createdBy = ExceptionsHelper.requireNonNull(createdBy, CREATED_BY);
         this.version = ExceptionsHelper.requireNonNull(version, VERSION);
@@ -147,7 +142,6 @@ public static TrainedModelConfig.Builder fromXContent(XContentParser parser, boo
         }
         this.estimatedOperations = estimatedOperations;
         this.licenseLevel = License.OperationMode.parse(ExceptionsHelper.requireNonNull(licenseLevel, LICENSE_LEVEL));
-        this.defaultFieldMap = defaultFieldMap == null ? null : Collections.unmodifiableMap(defaultFieldMap);
     }
 
     public TrainedModelConfig(StreamInput in) throws IOException {
@@ -163,13 +157,6 @@ public TrainedModelConfig(StreamInput in) throws IOException {
         estimatedHeapMemory = in.readVLong();
         estimatedOperations = in.readVLong();
         licenseLevel = License.OperationMode.parse(in.readString());
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-            this.defaultFieldMap = in.readBoolean() ?
-                Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString)) :
-                null;
-        } else {
-            this.defaultFieldMap = null;
-        }
     }
 
     public String getModelId() {
@@ -200,10 +187,6 @@ public Map<String, Object> getMetadata() {
         return metadata;
     }
 
-    public Map<String, String> getDefaultFieldMap() {
-        return defaultFieldMap;
-    }
-
     @Nullable
     public String getCompressedDefinition() throws IOException {
         if (definition == null) {
@@ -266,14 +249,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(estimatedHeapMemory);
         out.writeVLong(estimatedOperations);
         out.writeString(licenseLevel.description());
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-            if (defaultFieldMap != null) {
-                out.writeBoolean(true);
-                out.writeMap(defaultFieldMap, StreamOutput::writeString, StreamOutput::writeString);
-            } else {
-                out.writeBoolean(false);
-            }
-        }
     }
 
     @Override
@@ -308,9 +283,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             new ByteSizeValue(estimatedHeapMemory));
         builder.field(ESTIMATED_OPERATIONS.getPreferredName(), estimatedOperations);
         builder.field(LICENSE_LEVEL.getPreferredName(), licenseLevel.description());
-        if (defaultFieldMap != null && defaultFieldMap.isEmpty() == false) {
-            builder.field(DEFAULT_FIELD_MAP.getPreferredName(), defaultFieldMap);
-        }
         builder.endObject();
         return builder;
     }
@@ -336,7 +308,6 @@ public boolean equals(Object o) {
             Objects.equals(estimatedHeapMemory, that.estimatedHeapMemory) &&
             Objects.equals(estimatedOperations, that.estimatedOperations) &&
             Objects.equals(licenseLevel, that.licenseLevel) &&
-            Objects.equals(defaultFieldMap, that.defaultFieldMap) &&
             Objects.equals(metadata, that.metadata);
     }
 
@@ -353,8 +324,7 @@ public int hashCode() {
             estimatedHeapMemory,
             estimatedOperations,
             input,
-            licenseLevel,
-            defaultFieldMap);
+            licenseLevel);
     }
 
     public static class Builder {
@@ -371,7 +341,6 @@ public static class Builder {
         private Long estimatedOperations;
         private LazyModelDefinition definition;
         private String licenseLevel;
-        private Map<String, String> defaultFieldMap;
 
         public Builder() {}
 
@@ -388,7 +357,6 @@ public Builder(TrainedModelConfig config) {
             this.estimatedOperations = config.estimatedOperations;
             this.estimatedHeapMemory = config.estimatedHeapMemory;
             this.licenseLevel = config.licenseLevel.description();
-            this.defaultFieldMap = config.defaultFieldMap == null ? null : new HashMap<>(config.defaultFieldMap);
         }
 
         public Builder setModelId(String modelId) {
@@ -507,11 +475,6 @@ public Builder setLicenseLevel(String licenseLevel) {
             return this;
         }
 
-        public Builder setDefaultFieldMap(Map<String, String> defaultFieldMap) {
-            this.defaultFieldMap = defaultFieldMap;
-            return this;
-        }
-
         public Builder validate() {
             return validate(false);
         }
@@ -604,8 +567,7 @@ public TrainedModelConfig build() {
                 input,
                 estimatedHeapMemory == null ? 0 : estimatedHeapMemory,
                 estimatedOperations == null ? 0 : estimatedOperations,
-                licenseLevel == null ? License.OperationMode.PLATINUM.description() : licenseLevel,
-                defaultFieldMap);
+                licenseLevel == null ? License.OperationMode.PLATINUM.description() : licenseLevel);
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
index 4a11c361d9457..f15d8296bd531 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
@@ -53,10 +53,11 @@ private static ObjectParser<TrainedModelDefinition.Builder, Void> createParser(b
         ObjectParser<TrainedModelDefinition.Builder, Void> parser = new ObjectParser<>(NAME,
             ignoreUnknownFields,
             TrainedModelDefinition.Builder::builderForParser);
-        parser.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
+        parser.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
             (p, c, n) -> ignoreUnknownFields ?
                 p.namedObject(LenientlyParsedTrainedModel.class, n, null) :
                 p.namedObject(StrictlyParsedTrainedModel.class, n, null),
+            (modelDocBuilder) -> { /* Noop does not matter as we will throw if more than one is defined */ },
             TRAINED_MODEL);
         parser.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
             (p, c, n) -> ignoreUnknownFields ?
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
index 5955ecda6f276..5c9996ba21588 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -73,10 +73,11 @@ private static ObjectParser<Ensemble.Builder, Void> createParser(boolean lenient
                     p.namedObject(StrictlyParsedTrainedModel.class, n, null),
             (ensembleBuilder) -> ensembleBuilder.setModelsAreOrdered(true),
             TRAINED_MODELS);
-        parser.declareNamedObject(Ensemble.Builder::setOutputAggregator,
+        parser.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
             (p, c, n) ->
                 lenient ? p.namedObject(LenientlyParsedOutputAggregator.class, n, null) :
                     p.namedObject(StrictlyParsedOutputAggregator.class, n, null),
+            (ensembleBuilder) -> {/*Noop as it could be an array or object, it just has to be a one*/},
             AGGREGATE_OUTPUT);
         parser.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
         parser.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -413,6 +414,14 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
             return this;
         }
 
+        private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
+            if (outputAggregators.size() != 1) {
+                throw ExceptionsHelper.badRequestException("[{}] must have exactly one aggregator defined.",
+                    AGGREGATE_OUTPUT.getPreferredName());
+            }
+            this.setOutputAggregator(outputAggregators.get(0));
+        }
+
         private void setTargetType(String targetType) {
             this.targetType = TargetType.fromString(targetType);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
index faa921971a267..306ea243120e5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
@@ -13,6 +13,10 @@
 import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 
+import java.util.Comparator;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
+
 /**
  * Methods for handling index naming related functions
  */
@@ -23,7 +27,31 @@ public final class AnomalyDetectorsIndex {
     private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
     private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/";
 
-    private AnomalyDetectorsIndex() {}
+    // Visible for testing
+    static final Comparator<String> STATE_INDEX_NAME_COMPARATOR = new Comparator<>() {
+
+        private final Predicate<String> HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate();
+
+        @Override
+        public int compare(String index1, String index2) {
+            String[] index1Parts = index1.split("-");
+            String index1Suffix = index1Parts[index1Parts.length - 1];
+            boolean index1HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index1Suffix);
+            String[] index2Parts = index2.split("-");
+            String index2Suffix = index2Parts[index2Parts.length - 1];
+            boolean index2HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index2Suffix);
+            if (index1HasSixDigitsSuffix && index2HasSixDigitsSuffix) {
+                return index1Suffix.compareTo(index2Suffix);
+            } else if (index1HasSixDigitsSuffix != index2HasSixDigitsSuffix) {
+                return Boolean.compare(index1HasSixDigitsSuffix, index2HasSixDigitsSuffix);
+            } else {
+                return index1.compareTo(index2);
+            }
+        }
+    };
+
+    private AnomalyDetectorsIndex() {
+    }
 
     public static String jobResultsIndexPrefix() {
         return AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX;
@@ -80,13 +108,8 @@ public static String configIndexName() {
      */
     public static void createStateIndexAndAliasIfNecessary(Client client, ClusterState state, IndexNameExpressionResolver resolver,
                                                            final ActionListener<Boolean> finalListener) {
-        MlIndexAndAlias.createIndexAndAliasIfNecessary(
-            client,
-            state,
-            resolver,
-            AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
-            AnomalyDetectorsIndex.jobStateIndexWriteAlias(),
-            finalListener);
+        MlIndexAndAlias.createIndexAndAliasIfNecessary(client, state, resolver,
+            AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias(), finalListener);
     }
 
     public static String resultsMapping() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
index 6a06608638ff9..9ae1aa2957adf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
@@ -317,13 +317,12 @@ public final class ReservedFieldNames {
             Classification.NAME.getPreferredName(),
             Classification.DEPENDENT_VARIABLE.getPreferredName(),
             Classification.PREDICTION_FIELD_NAME.getPreferredName(),
-            Classification.CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(),
             Classification.NUM_TOP_CLASSES.getPreferredName(),
             Classification.TRAINING_PERCENT.getPreferredName(),
             BoostedTreeParams.LAMBDA.getPreferredName(),
             BoostedTreeParams.GAMMA.getPreferredName(),
             BoostedTreeParams.ETA.getPreferredName(),
-            BoostedTreeParams.MAX_TREES.getPreferredName(),
+            BoostedTreeParams.MAXIMUM_NUMBER_TREES.getPreferredName(),
             BoostedTreeParams.FEATURE_BAG_FRACTION.getPreferredName(),
             BoostedTreeParams.NUM_TOP_FEATURE_IMPORTANCE_VALUES.getPreferredName(),
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
index 2c3aba673c6d0..79b9987d35193 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
@@ -5,27 +5,20 @@
  */
 package org.elasticsearch.xpack.core.ml.utils;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.Nullable;
 
 import java.util.Arrays;
 import java.util.Comparator;
-import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
@@ -37,8 +30,6 @@
  */
 public final class MlIndexAndAlias {
 
-    private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class);
-
     // Visible for testing
     static final Comparator<String> INDEX_NAME_COMPARATOR = new Comparator<>() {
 
@@ -70,114 +61,60 @@ private MlIndexAndAlias() {}
      * or to the index with the highest suffix if the index did not have to be created.
      * The listener is notified with a {@code boolean} that informs whether the index or the alias were created.
      */
-    public static void createIndexAndAliasIfNecessary(Client client,
-                                                      ClusterState clusterState,
-                                                      IndexNameExpressionResolver resolver,
-                                                      String indexPatternPrefix,
-                                                      String alias,
-                                                      ActionListener<Boolean> listener) {
-
-        String legacyIndexWithoutSuffix = indexPatternPrefix;
-        String indexPattern = indexPatternPrefix + "*";
-        // The initial index name must be suitable for rollover functionality.
-        String firstConcreteIndex = indexPatternPrefix + "-000001";
-        String[] concreteIndexNames =
-            resolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), indexPattern);
-        Optional<IndexMetaData> indexPointedByCurrentWriteAlias = clusterState.getMetaData().hasAlias(alias)
-            ? clusterState.getMetaData().getAliasAndIndexLookup().get(alias).getIndices().stream().findFirst()
-            : Optional.empty();
-
-        if (concreteIndexNames.length == 0) {
-            if (indexPointedByCurrentWriteAlias.isEmpty()) {
-                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
-                return;
-            }
-            logger.error(
-                "There are no indices matching '{}' pattern but '{}' alias points at [{}]. This should never happen.",
-                indexPattern, alias, indexPointedByCurrentWriteAlias.get());
-        } else if (concreteIndexNames.length == 1 && concreteIndexNames[0].equals(legacyIndexWithoutSuffix)) {
-            if (indexPointedByCurrentWriteAlias.isEmpty()) {
-                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
-                return;
-            }
-            if (indexPointedByCurrentWriteAlias.get().getIndex().getName().equals(legacyIndexWithoutSuffix)) {
-                createFirstConcreteIndex(
-                    client,
-                    firstConcreteIndex,
-                    alias,
-                    false,
-                    ActionListener.wrap(
-                        unused -> updateWriteAlias(client, alias, legacyIndexWithoutSuffix, firstConcreteIndex, listener),
-                        listener::onFailure)
-                );
-                return;
-            }
-            logger.error(
-                "There is exactly one index (i.e. '{}') matching '{}' pattern but '{}' alias points at [{}]. This should never happen.",
-                legacyIndexWithoutSuffix, indexPattern, alias, indexPointedByCurrentWriteAlias.get());
-        } else {
-            if (indexPointedByCurrentWriteAlias.isEmpty()) {
-                assert concreteIndexNames.length > 0;
-                String latestConcreteIndexName = Arrays.stream(concreteIndexNames).max(INDEX_NAME_COMPARATOR).get();
-                updateWriteAlias(client, alias, null, latestConcreteIndexName, listener);
-                return;
-            }
+    public static void createIndexAndAliasIfNecessary(Client client, ClusterState clusterState, IndexNameExpressionResolver resolver,
+                                                      String indexPatternPrefix, String alias, ActionListener<Boolean> listener) {
+        if (clusterState.getMetaData().getAliasAndIndexLookup().containsKey(alias)) {
+            listener.onResponse(false);
+            return;
         }
-        // If the alias is set, there is nothing more to do.
-        listener.onResponse(false);
-    }
-
-    private static void createFirstConcreteIndex(Client client,
-                                                 String index,
-                                                 String alias,
-                                                 boolean addAlias,
-                                                 ActionListener<Boolean> listener) {
-        CreateIndexRequestBuilder requestBuilder = client.admin()
-            .indices()
-            .prepareCreate(index);
-        if (addAlias) {
-            requestBuilder.addAlias(new Alias(alias));
-        }
-        CreateIndexRequest request = requestBuilder.request();
 
-        executeAsyncWithOrigin(client.threadPool().getThreadContext(),
-            ML_ORIGIN,
-            request,
-            ActionListener.<CreateIndexResponse>wrap(
-                createIndexResponse -> listener.onResponse(true),
-                createIndexFailure -> {
-                    // If it was created between our last check, and this request being handled, we should add the alias
-                    // Adding an alias that already exists is idempotent. So, no need to double check if the alias exists
-                    // as well.
-                    if (ExceptionsHelper.unwrapCause(createIndexFailure) instanceof ResourceAlreadyExistsException) {
-                        updateWriteAlias(client, alias, null, index, listener);
-                    } else {
-                        listener.onFailure(createIndexFailure);
-                    }
-                }),
-            client.admin().indices()::create);
-    }
-
-    private static void updateWriteAlias(Client client,
-                                         String alias,
-                                         @Nullable String currentIndex,
-                                         String newIndex,
-                                         ActionListener<Boolean> listener) {
-        IndicesAliasesRequestBuilder requestBuilder = client.admin()
-            .indices()
-            .prepareAliases()
-            .addAlias(newIndex, alias);
-        if (currentIndex != null) {
-            requestBuilder.removeAlias(currentIndex, alias);
+        final ActionListener<String> createAliasListener = ActionListener.wrap(
+            concreteIndexName -> {
+                final IndicesAliasesRequest request = client.admin()
+                    .indices()
+                    .prepareAliases()
+                    .addAlias(concreteIndexName, alias)
+                    .request();
+                executeAsyncWithOrigin(client.threadPool().getThreadContext(),
+                    ML_ORIGIN,
+                    request,
+                    ActionListener.<AcknowledgedResponse>wrap(
+                        resp -> listener.onResponse(resp.isAcknowledged()),
+                        listener::onFailure),
+                    client.admin().indices()::aliases);
+            },
+            listener::onFailure
+        );
+
+        String[] stateIndices = resolver.concreteIndexNames(clusterState,
+            IndicesOptions.lenientExpandOpen(), indexPatternPrefix + "*");
+        if (stateIndices.length > 0) {
+            String latestStateIndex = Arrays.stream(stateIndices).max(INDEX_NAME_COMPARATOR).get();
+            createAliasListener.onResponse(latestStateIndex);
+        } else {
+            // The initial index name must be suitable for rollover functionality.
+            String initialJobStateIndex = indexPatternPrefix + "-000001";
+            CreateIndexRequest createIndexRequest = client.admin()
+                .indices()
+                .prepareCreate(initialJobStateIndex)
+                .addAlias(new Alias(alias))
+                .request();
+            executeAsyncWithOrigin(client.threadPool().getThreadContext(),
+                ML_ORIGIN,
+                createIndexRequest,
+                ActionListener.<CreateIndexResponse>wrap(
+                    createIndexResponse -> listener.onResponse(true),
+                    createIndexFailure -> {
+                        // If it was created between our last check, and this request being handled, we should add the alias
+                        // Adding an alias that already exists is idempotent. So, no need to double check if the alias exists
+                        // as well.
+                        if (ExceptionsHelper.unwrapCause(createIndexFailure) instanceof ResourceAlreadyExistsException) {
+                            createAliasListener.onResponse(initialJobStateIndex);
+                        } else {
+                            listener.onFailure(createIndexFailure);
+                        }
+                    }),
+                client.admin().indices()::create);
         }
-        IndicesAliasesRequest request = requestBuilder.request();
-
-        executeAsyncWithOrigin(client.threadPool().getThreadContext(),
-            ML_ORIGIN,
-            request,
-            ActionListener.<AcknowledgedResponse>wrap(
-                resp -> listener.onResponse(resp.isAcknowledged()),
-                listener::onFailure),
-            client.admin().indices()::aliases);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
deleted file mode 100644
index 9828635ae64fe..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.search.action;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.rest.RestStatus;
-
-import java.io.IOException;
-
-import static org.elasticsearch.rest.RestStatus.OK;
-
-/**
- * A response of an async search request.
- */
-public class AsyncSearchResponse extends ActionResponse implements StatusToXContentObject {
-    @Nullable
-    private final String id;
-    private final int version;
-    private final SearchResponse searchResponse;
-    private final ElasticsearchException error;
-    private final boolean isRunning;
-    private final boolean isPartial;
-
-    private final long startTimeMillis;
-    private final long expirationTimeMillis;
-
-    /**
-     * Creates an {@link AsyncSearchResponse} with meta-information only (not-modified).
-     */
-    public AsyncSearchResponse(String id,
-                               int version,
-                               boolean isPartial,
-                               boolean isRunning,
-                               long startTimeMillis,
-                               long expirationTimeMillis) {
-        this(id, version, null, null, isPartial, isRunning, startTimeMillis, expirationTimeMillis);
-    }
-
-    /**
-     * Creates a new {@link AsyncSearchResponse}
-     *
-     * @param id The id of the search for further retrieval, <code>null</code> if not stored.
-     * @param version The version number of this response.
-     * @param searchResponse The actual search response.
-     * @param error The error if the search failed, <code>null</code> if the search is running
-     *                or has completed without failure.
-     * @param isPartial Whether the <code>searchResponse</code> contains partial results.
-     * @param isRunning Whether the search is running in the cluster.
-     * @param startTimeMillis The start date of the search in milliseconds since epoch.
-     */
-    public AsyncSearchResponse(String id,
-                               int version,
-                               SearchResponse searchResponse,
-                               ElasticsearchException error,
-                               boolean isPartial,
-                               boolean isRunning,
-                               long startTimeMillis,
-                               long expirationTimeMillis) {
-        this.id = id;
-        this.version = version;
-        this.error = error;
-        this.searchResponse = searchResponse;
-        this.isPartial = isPartial;
-        this.isRunning = isRunning;
-        this.startTimeMillis = startTimeMillis;
-        this.expirationTimeMillis = expirationTimeMillis;
-    }
-
-    public AsyncSearchResponse(StreamInput in) throws IOException {
-        this.id = in.readOptionalString();
-        this.version = in.readVInt();
-        this.error = in.readOptionalWriteable(ElasticsearchException::new);
-        this.searchResponse = in.readOptionalWriteable(SearchResponse::new);
-        this.isPartial = in.readBoolean();
-        this.isRunning = in.readBoolean();
-        this.startTimeMillis = in.readLong();
-        this.expirationTimeMillis = in.readLong();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeOptionalString(id);
-        out.writeVInt(version);
-        out.writeOptionalWriteable(error);
-        out.writeOptionalWriteable(searchResponse);
-        out.writeBoolean(isPartial);
-        out.writeBoolean(isRunning);
-        out.writeLong(startTimeMillis);
-        out.writeLong(expirationTimeMillis);
-    }
-
-    public AsyncSearchResponse clone(String id) {
-        return new AsyncSearchResponse(id, version, searchResponse, error, isPartial, false, startTimeMillis, expirationTimeMillis);
-    }
-
-    /**
-     * Returns the id of the async search request or null if the response is not stored in the cluster.
-     */
-    @Nullable
-    public String getId() {
-        return id;
-    }
-
-    /**
-     * Returns the version of this response.
-     */
-    public int getVersion() {
-        return version;
-    }
-
-    /**
-     * Returns the current {@link SearchResponse} or <code>null</code> if not available.
-     *
-     * See {@link #isPartial()} to determine whether the response contains partial or complete
-     * results.
-     */
-    public SearchResponse getSearchResponse() {
-        return searchResponse;
-    }
-
-    /**
-     * Returns the failure reason or null if the query is running or has completed normally.
-     */
-    public ElasticsearchException getFailure() {
-        return error;
-    }
-
-    /**
-     * Returns <code>true</code> if the {@link SearchResponse} contains partial
-     * results computed from a subset of the total shards.
-     */
-    public boolean isPartial() {
-        return isPartial;
-    }
-
-    /**
-     * Whether the search is still running in the cluster.
-     *
-     * A value of <code>false</code> indicates that the response is final
-     * even if {@link #isPartial()} returns <code>true</code>. In such case,
-     * the partial response represents the status of the search before a
-     * non-recoverable failure.
-     */
-    public boolean isRunning() {
-        return isRunning;
-    }
-
-    /**
-     * When this response was created as a timestamp in milliseconds since epoch.
-     */
-    public long getStartTime() {
-        return startTimeMillis;
-    }
-
-    /**
-     * When this response will expired as a timestamp in milliseconds since epoch.
-     */
-    public long getExpirationTime() {
-        return expirationTimeMillis;
-    }
-
-    @Override
-    public RestStatus status() {
-        if (searchResponse == null || isPartial) {
-            // shard failures are not considered fatal for partial results so
-            // we return OK until we get the final response even if we don't have
-            // a single successful shard.
-            return error != null ? error.status() : OK;
-        } else {
-            return searchResponse.status();
-        }
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        if (id != null) {
-            builder.field("id", id);
-        }
-        builder.field("version", version);
-        builder.field("is_partial", isPartial);
-        builder.field("is_running", isRunning);
-        builder.field("start_time_in_millis", startTimeMillis);
-        builder.field("expiration_time_in_millis", expirationTimeMillis);
-
-        if (searchResponse != null) {
-            builder.field("response");
-            searchResponse.toXContent(builder, params);
-        }
-        if (error != null) {
-            builder.startObject("error");
-            error.toXContent(builder, params);
-            builder.endObject();
-        }
-        builder.endObject();
-        return builder;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java
deleted file mode 100644
index d69de80d2293e..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.search.action;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-
-import java.io.IOException;
-import java.util.Objects;
-
-public class DeleteAsyncSearchAction extends ActionType<AcknowledgedResponse> {
-    public static final DeleteAsyncSearchAction INSTANCE = new DeleteAsyncSearchAction();
-    public static final String NAME = "indices:data/read/async_search/delete";
-
-    private DeleteAsyncSearchAction() {
-        super(NAME, AcknowledgedResponse::new);
-    }
-
-    @Override
-    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
-        return AcknowledgedResponse::new;
-    }
-
-    public static class Request extends ActionRequest {
-        private final String id;
-
-        public Request(String id) {
-            this.id = id;
-        }
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            this.id = in.readString();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeString(id);
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            return null;
-        }
-
-        public String getId() {
-            return id;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Request request = (Request) o;
-            return id.equals(request.id);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(id);
-        }
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java
deleted file mode 100644
index fe4801aab4a10..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.search.action;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.unit.TimeValue;
-
-import java.io.IOException;
-import java.util.Objects;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest.MIN_KEEP_ALIVE;
-
-public class GetAsyncSearchAction extends ActionType<AsyncSearchResponse> {
-    public static final GetAsyncSearchAction INSTANCE = new GetAsyncSearchAction();
-    public static final String NAME = "indices:data/read/async_search/get";
-
-    private GetAsyncSearchAction() {
-        super(NAME, AsyncSearchResponse::new);
-    }
-
-    @Override
-    public Writeable.Reader<AsyncSearchResponse> getResponseReader() {
-        return AsyncSearchResponse::new;
-    }
-
-    public static class Request extends ActionRequest {
-        private final String id;
-        private TimeValue waitForCompletion = TimeValue.MINUS_ONE;
-        private TimeValue keepAlive = TimeValue.MINUS_ONE;
-
-        /**
-         * Creates a new request
-         *
-         * @param id The id of the search progress request.
-         */
-        public Request(String id) {
-            this.id = id;
-        }
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            this.id = in.readString();
-            this.waitForCompletion = TimeValue.timeValueMillis(in.readLong());
-            this.keepAlive = in.readTimeValue();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeString(id);
-            out.writeLong(waitForCompletion.millis());
-            out.writeTimeValue(keepAlive);
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            ActionRequestValidationException validationException = null;
-            if (keepAlive.getMillis() != -1 && keepAlive.getMillis() < MIN_KEEP_ALIVE) {
-                validationException =
-                    addValidationError("keep_alive must be greater than 1 minute, got:" + keepAlive.toString(), validationException);
-            }
-            return validationException;
-        }
-
-        /**
-         * Returns the id of the async search.
-         */
-        public String getId() {
-            return id;
-        }
-
-        /**
-         * Sets the minimum time that the request should wait before returning a partial result (defaults to no wait).
-         */
-        public Request setWaitForCompletion(TimeValue timeValue) {
-            this.waitForCompletion = timeValue;
-            return this;
-        }
-
-        public TimeValue getWaitForCompletion() {
-            return waitForCompletion;
-        }
-
-        /**
-         * Extends the amount of time after which the result will expire (defaults to no extension).
-         */
-        public Request setKeepAlive(TimeValue timeValue) {
-            this.keepAlive = timeValue;
-            return this;
-        }
-
-        public TimeValue getKeepAlive() {
-            return keepAlive;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Request request = (Request) o;
-            return Objects.equals(id, request.id) &&
-                waitForCompletion.equals(request.waitForCompletion) &&
-                keepAlive.equals(request.keepAlive);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(id, waitForCompletion, keepAlive);
-        }
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java
deleted file mode 100644
index a91c861c64e16..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.search.action;
-
-import org.elasticsearch.action.ActionType;
-
-public final class SubmitAsyncSearchAction extends ActionType<AsyncSearchResponse> {
-    public static final SubmitAsyncSearchAction INSTANCE = new SubmitAsyncSearchAction();
-    public static final String NAME = "indices:data/read/async_search/submit";
-
-    private SubmitAsyncSearchAction() {
-        super(NAME, AsyncSearchResponse::new);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
deleted file mode 100644
index a6397cbf08c3b..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.search.action;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.tasks.CancellableTask;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskId;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
-/**
- * A request to track asynchronously the progress of a search against one or more indices.
- *
- * @see AsyncSearchResponse
- */
-public class SubmitAsyncSearchRequest extends ActionRequest {
-    public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
-
-    private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1);
-    private boolean cleanOnCompletion = true;
-    private TimeValue keepAlive = TimeValue.timeValueDays(5);
-
-    private final SearchRequest request;
-
-    /**
-     * Creates a new request
-     */
-    public SubmitAsyncSearchRequest(String... indices) {
-        this(new SearchSourceBuilder(), indices);
-    }
-
-    /**
-     * Creates a new request
-     */
-    public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) {
-        this.request = new SearchRequest(indices, source);
-        request.setCcsMinimizeRoundtrips(false);
-        request.setPreFilterShardSize(1);
-        request.setBatchedReduceSize(5);
-        request.requestCache(true);
-    }
-
-    public SubmitAsyncSearchRequest(StreamInput in) throws IOException {
-        this.request = new SearchRequest(in);
-        this.waitForCompletion = in.readTimeValue();
-        this.keepAlive = in.readTimeValue();
-        this.cleanOnCompletion = in.readBoolean();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        request.writeTo(out);
-        out.writeTimeValue(waitForCompletion);
-        out.writeTimeValue(keepAlive);
-        out.writeBoolean(cleanOnCompletion);
-    }
-
-    /**
-     * Sets the number of shard results that should be returned to notify search progress (default to 5).
-     */
-    public SubmitAsyncSearchRequest setBatchedReduceSize(int size) {
-        request.setBatchedReduceSize(size);
-        return this;
-    }
-
-    public int getBatchReduceSize() {
-        return request.getBatchedReduceSize();
-    }
-
-    /**
-     * Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second).
-     */
-    public SubmitAsyncSearchRequest setWaitForCompletion(TimeValue waitForCompletion) {
-        this.waitForCompletion = waitForCompletion;
-        return this;
-    }
-
-    public TimeValue getWaitForCompletion() {
-        return waitForCompletion;
-    }
-
-    /**
-     * Sets the amount of time after which the result will expire (defaults to 5 days).
-     */
-    public SubmitAsyncSearchRequest setKeepAlive(TimeValue keepAlive) {
-        this.keepAlive = keepAlive;
-        return this;
-    }
-
-    public TimeValue getKeepAlive() {
-        return keepAlive;
-    }
-
-    /**
-     * Returns the underlying {@link SearchRequest}.
-     */
-    public SearchRequest getSearchRequest() {
-        return request;
-    }
-
-    /**
-     * Should the resource be removed on completion or failure (defaults to true).
-     */
-    public SubmitAsyncSearchRequest setCleanOnCompletion(boolean value) {
-        this.cleanOnCompletion = value;
-        return this;
-    }
-
-    public boolean isCleanOnCompletion() {
-        return cleanOnCompletion;
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = request.validate();
-        if (request.scroll() != null) {
-            addValidationError("[scroll] queries are not supported", validationException);
-        }
-        if (request.isSuggestOnly()) {
-            validationException = addValidationError("suggest-only queries are not supported", validationException);
-        }
-        if (keepAlive.getMillis() < MIN_KEEP_ALIVE) {
-            validationException =
-                addValidationError("[keep_alive] must be greater than 1 minute, got:" + keepAlive.toString(), validationException);
-        }
-        if (request.isCcsMinimizeRoundtrips()) {
-            validationException =
-                addValidationError("[ccs_minimize_roundtrips] is not supported on async search queries", validationException);
-        }
-
-        return validationException;
-    }
-
-    @Override
-    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
-        return new CancellableTask(id, type, action, toString(), parentTaskId, headers) {
-            @Override
-            public boolean shouldCancelChildrenOnCancellation() {
-                return true;
-            }
-        };
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        SubmitAsyncSearchRequest request1 = (SubmitAsyncSearchRequest) o;
-        return cleanOnCompletion == request1.cleanOnCompletion &&
-            waitForCompletion.equals(request1.waitForCompletion) &&
-            keepAlive.equals(request1.keepAlive) &&
-            request.equals(request1.request);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(waitForCompletion, cleanOnCompletion, keepAlive, request);
-    }
-
-    @Override
-    public String toString() {
-        return "SubmitAsyncSearchRequest{" +
-            "waitForCompletion=" + waitForCompletion +
-            ", cleanOnCompletion=" + cleanOnCompletion +
-            ", keepAlive=" + keepAlive +
-            ", request=" + request +
-            '}';
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
index 67307ad88cf3b..c31dbeb7e4859 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
@@ -74,15 +74,11 @@ public CreateApiKeyRequestBuilder source(BytesReference source, XContentType xCo
         final NamedXContentRegistry registry = NamedXContentRegistry.EMPTY;
         try (InputStream stream = source.streamInput();
                 XContentParser parser = xContentType.xContent().createParser(registry, LoggingDeprecationHandler.INSTANCE, stream)) {
-            CreateApiKeyRequest createApiKeyRequest = parse(parser);
+            CreateApiKeyRequest createApiKeyRequest = PARSER.parse(parser, null);
             setName(createApiKeyRequest.getName());
             setRoleDescriptors(createApiKeyRequest.getRoleDescriptors());
             setExpiration(createApiKeyRequest.getExpiration());
         }
         return this;
     }
-
-    public static CreateApiKeyRequest parse(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java
deleted file mode 100644
index 07e5c3bbf2521..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.core.security.action;
-
-import org.elasticsearch.action.ActionType;
-
-/**
- * ActionType for the creation of an API key on behalf of another user
- * This returns the {@link CreateApiKeyResponse} because the REST output is intended to be identical to the {@link CreateApiKeyAction}.
- */
-public final class GrantApiKeyAction extends ActionType<CreateApiKeyResponse> {
-
-    public static final String NAME = "cluster:admin/xpack/security/api_key/grant";
-    public static final GrantApiKeyAction INSTANCE = new GrantApiKeyAction();
-
-    private GrantApiKeyAction() {
-        super(NAME, CreateApiKeyResponse::new);
-    }
-
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java
deleted file mode 100644
index 4daf3c84df61c..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.core.security.action;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.SecureString;
-
-import java.io.IOException;
-import java.util.Objects;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
-/**
- * Request class used for the creation of an API key on behalf of another user.
- * Logically this is similar to {@link CreateApiKeyRequest}, but is for cases when the user that has permission to call this action
- * is different to the user for whom the API key should be created
- */
-public final class GrantApiKeyRequest extends ActionRequest {
-
-    public static final String PASSWORD_GRANT_TYPE = "password";
-    public static final String ACCESS_TOKEN_GRANT_TYPE = "access_token";
-
-    /**
-     * Fields related to the end user authentication
-     */
-    public static class Grant implements Writeable {
-        private String type;
-        private String username;
-        private SecureString password;
-        private SecureString accessToken;
-
-        public Grant() {
-        }
-
-        public Grant(StreamInput in) throws IOException {
-            this.type = in.readString();
-            this.username = in.readOptionalString();
-            this.password = in.readOptionalSecureString();
-            this.accessToken = in.readOptionalSecureString();
-        }
-
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(type);
-            out.writeOptionalString(username);
-            out.writeOptionalSecureString(password);
-            out.writeOptionalSecureString(accessToken);
-        }
-
-        public String getType() {
-            return type;
-        }
-
-        public String getUsername() {
-            return username;
-        }
-
-        public SecureString getPassword() {
-            return password;
-        }
-
-        public SecureString getAccessToken() {
-            return accessToken;
-        }
-
-        public void setType(String type) {
-            this.type = type;
-        }
-
-        public void setUsername(String username) {
-            this.username = username;
-        }
-
-        public void setPassword(SecureString password) {
-            this.password = password;
-        }
-
-        public void setAccessToken(SecureString accessToken) {
-            this.accessToken = accessToken;
-        }
-    }
-
-    private final Grant grant;
-    private CreateApiKeyRequest apiKey;
-
-    public GrantApiKeyRequest() {
-        this.grant = new Grant();
-        this.apiKey = new CreateApiKeyRequest();
-    }
-
-    public GrantApiKeyRequest(StreamInput in) throws IOException {
-        super(in);
-        this.grant = new Grant(in);
-        this.apiKey = new CreateApiKeyRequest(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        grant.writeTo(out);
-        apiKey.writeTo(out);
-    }
-
-    public WriteRequest.RefreshPolicy getRefreshPolicy() {
-        return apiKey.getRefreshPolicy();
-    }
-
-    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
-        apiKey.setRefreshPolicy(refreshPolicy);
-    }
-
-    public Grant getGrant() {
-        return grant;
-    }
-
-    public CreateApiKeyRequest getApiKeyRequest() {
-        return apiKey;
-    }
-
-    public void setApiKeyRequest(CreateApiKeyRequest apiKeyRequest) {
-        this.apiKey = Objects.requireNonNull(apiKeyRequest, "Cannot set a null api_key");
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = apiKey.validate();
-        if (grant.type == null) {
-            validationException = addValidationError("[grant_type] is required", validationException);
-        } else if (grant.type.equals(PASSWORD_GRANT_TYPE)) {
-            validationException = validateRequiredField("username", grant.username, validationException);
-            validationException = validateRequiredField("password", grant.password, validationException);
-            validationException = validateUnsupportedField("access_token", grant.accessToken, validationException);
-        } else if (grant.type.equals(ACCESS_TOKEN_GRANT_TYPE)) {
-            validationException = validateRequiredField("access_token", grant.accessToken, validationException);
-            validationException = validateUnsupportedField("username", grant.username, validationException);
-            validationException = validateUnsupportedField("password", grant.password, validationException);
-        } else {
-            validationException = addValidationError("grant_type [" + grant.type + "] is not supported", validationException);
-        }
-        return validationException;
-    }
-
-    private ActionRequestValidationException validateRequiredField(String fieldName, CharSequence fieldValue,
-                                                                   ActionRequestValidationException validationException) {
-        if (fieldValue == null || fieldValue.length() == 0) {
-            return addValidationError("[" + fieldName + "] is required for grant_type [" + grant.type + "]", validationException);
-        }
-        return validationException;
-    }
-
-    private ActionRequestValidationException validateUnsupportedField(String fieldName, CharSequence fieldValue,
-                                                                      ActionRequestValidationException validationException) {
-        if (fieldValue != null && fieldValue.length() > 0) {
-            return addValidationError("[" + fieldName + "] is not supported for grant_type [" + grant.type + "]", validationException);
-        }
-        return validationException;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java
index 2d7f660bd188e..77f6c537b6f81 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java
@@ -23,7 +23,7 @@ public final class RestrictedIndicesNames {
     public static final String SECURITY_TOKENS_ALIAS = ".security-tokens";
 
     // public for tests
-    public static final String ASYNC_SEARCH_PREFIX = ".async-search";
+    public static final String ASYNC_SEARCH_PREFIX = ".async-search-";
     private static final Automaton ASYNC_SEARCH_AUTOMATON = Automatons.patterns(ASYNC_SEARCH_PREFIX + "*");
 
     // public for tests
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
index 0d7ab44ed9582..0ef7912c58017 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
@@ -254,10 +254,10 @@ public String generateSnapshotName(Context context) {
      */
     public CreateSnapshotRequest toRequest() {
         CreateSnapshotRequest req = new CreateSnapshotRequest(repository, generateSnapshotName(new ResolverContext()));
-        Map<String, Object> mergedConfiguration = configuration == null ? new HashMap<>() : new HashMap<>(configuration);
         @SuppressWarnings("unchecked")
-        Map<String, Object> metadata = (Map<String, Object>) mergedConfiguration.get("metadata");
+        Map<String, Object> metadata = (Map<String, Object>) configuration.get("metadata");
         Map<String, Object> metadataWithAddedPolicyName = addPolicyNameToMetadata(metadata);
+        Map<String, Object> mergedConfiguration = new HashMap<>(configuration);
         mergedConfiguration.put("metadata", metadataWithAddedPolicyName);
         req.source(mergedConfiguration);
         req.waitForCompletion(true);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java
index bd215126119d5..5686f3c9377e1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java
@@ -80,7 +80,7 @@ public SingleGroupSource(final String field, final ScriptConfig scriptConfig) {
 
     public SingleGroupSource(StreamInput in) throws IOException {
         field = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // change to 7.7.0
             scriptConfig = in.readOptionalWriteable(ScriptConfig::new);
         } else {
             scriptConfig = null;
@@ -107,7 +107,7 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(field);
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
             out.writeOptionalWriteable(scriptConfig);
         }
     }
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json
index e3cd7f51dc43a..fb0b7ad6c2bd5 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json
@@ -9,7 +9,6 @@
       "auto_expand_replicas" : "0-1",
       "hidden": true
     }
-    ${xpack.ml.index.lifecycle.settings}
   },
   "mappings" : {
     "_doc": {
@@ -19,5 +18,5 @@
       "enabled": false
     }
   },
-  "aliases" : {}
+  "aliases" : { }
 }
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
index fe0c559b12245..1abd7e42e8ad1 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
@@ -40,12 +40,9 @@
               "lambda" : {
                 "type" : "double"
               },
-              "max_trees" : {
+              "maximum_number_trees" : {
                 "type" : "integer"
               },
-              "class_assignment_objective" : {
-                "type" : "keyword"
-              },
               "num_top_classes" : {
                 "type" : "integer"
               },
@@ -90,7 +87,7 @@
               "lambda" : {
                 "type" : "double"
               },
-              "max_trees" : {
+              "maximum_number_trees" : {
                 "type" : "integer"
               },
               "num_top_feature_importance_values" : {
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json
index 2a1131ca557a9..5cbee23ca8094 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json
@@ -64,9 +64,6 @@
         },
         "total_definition_length": {
           "type": "long"
-        },
-        "default_field_map": {
-          "enabled": false
         }
       }
     }
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json
deleted file mode 100644
index 09a8db4e1ee9d..0000000000000
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-  "phases": {
-    "hot": {
-      "actions": {
-        "rollover": {
-          "max_size": "50GB"
-        }
-      }
-    }
-  }
-}
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json
index 223d1a79bad75..1c694d9d1a7e5 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json
@@ -10,8 +10,6 @@
       "auto_expand_replicas" : "0-1",
       "hidden": true
     }
-    ${xpack.ml.index.lifecycle.settings}
   },
-  "mappings" : ${xpack.ml.stats.mappings},
-  "aliases" : {}
+  "mappings" : ${xpack.ml.stats.mappings}
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
index c3958f31563d0..ef4d35401675b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
@@ -105,9 +105,9 @@ public void testSnapshotAndRestore() throws Exception {
         });
         assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
 
-        // can-match phase pre-filters access to non-existing field
-        assertEquals(0,
-            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get().getHits().getTotalHits().value);
+        e = expectThrows(SearchPhaseExecutionException.class, () ->
+            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get());
+        assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
         // make sure deletes do not work
         String idToDelete = "" + randomIntBetween(0, builders.length);
         expectThrows(ClusterBlockException.class, () -> client().prepareDelete(sourceIdx, idToDelete)
@@ -130,9 +130,9 @@ public void testSnapshotAndRestoreWithNested() throws Exception {
         SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () ->
             client().prepareSearch(sourceIdx).setQuery(QueryBuilders.idsQuery().addIds("" + randomIntBetween(0, builders.length))).get());
         assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
-        // can-match phase pre-filters access to non-existing field
-        assertEquals(0,
-            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get().getHits().getTotalHits().value);
+        e = expectThrows(SearchPhaseExecutionException.class, () ->
+            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get());
+        assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
         // make sure deletes do not work
         String idToDelete = "" + randomIntBetween(0, builders.length);
         expectThrows(ClusterBlockException.class, () -> client().prepareDelete(sourceIdx, idToDelete)
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
index 8dad3cf3a4a81..8cd8a2ef0b36d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.protocol.xpack.frozen.FreezeRequest;
-import org.elasticsearch.protocol.xpack.frozen.FreezeResponse;
 import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
 import org.elasticsearch.xpack.core.ilm.Step.StepKey;
 import org.mockito.Mockito;
@@ -74,7 +73,7 @@ public void testFreeze() {
             assertNotNull(request);
             assertEquals(1, request.indices().length);
             assertEquals(indexMetaData.getIndex().getName(), request.indices()[0]);
-            listener.onResponse(new FreezeResponse(true, true));
+            listener.onResponse(null);
             return null;
         }).when(indicesClient).execute(Mockito.any(), Mockito.any(), Mockito.any());
 
@@ -128,32 +127,4 @@ public void onFailure(Exception e) {
 
         assertThat(exceptionThrown.get(), equalTo(true));
     }
-
-    public void testNotAcknowledged() {
-        IndexMetaData indexMetaData = getIndexMetaData();
-
-        Mockito.doAnswer(invocation -> {
-            @SuppressWarnings("unchecked")
-            ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[2];
-            listener.onResponse(new FreezeResponse(false, false));
-            return null;
-        }).when(indicesClient).execute(Mockito.any(), Mockito.any(), Mockito.any());
-
-        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
-        FreezeStep step = createRandomInstance();
-        step.performAction(indexMetaData, emptyClusterState(), null, new AsyncActionStep.Listener() {
-            @Override
-            public void onResponse(boolean complete) {
-                throw new AssertionError("Unexpected method call");
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                assertEquals("freeze index request failed to be acknowledged", e.getMessage());
-                exceptionThrown.set(true);
-            }
-        });
-
-        assertThat(exceptionThrown.get(), equalTo(true));
-    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
index 1c733e04504c8..7dd0f6f9b0ca9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
@@ -155,14 +155,12 @@ protected void assertOnBWCObject(DataFrameAnalyticsConfig bwcSerializedObject, D
             bwcAnalysis = new Classification(bwcClassification.getDependentVariable(),
                 bwcClassification.getBoostedTreeParams(),
                 bwcClassification.getPredictionFieldName(),
-                bwcClassification.getClassAssignmentObjective(),
                 bwcClassification.getNumTopClasses(),
                 bwcClassification.getTrainingPercent(),
                 42L);
             testAnalysis = new Classification(testClassification.getDependentVariable(),
                 testClassification.getBoostedTreeParams(),
                 testClassification.getPredictionFieldName(),
-                testClassification.getClassAssignmentObjective(),
                 testClassification.getNumTopClasses(),
                 testClassification.getTrainingPercent(),
                 42L);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
index f1114bf3445a8..7f9d8c243e50b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
@@ -39,7 +39,7 @@ public static BoostedTreeParams createRandom() {
             .setLambda(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setGamma(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setEta(randomBoolean() ? null : randomDoubleBetween(0.001, 1.0, true))
-            .setMaxTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
+            .setMaximumNumberTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
             .setFeatureBagFraction(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, false))
             .setNumTopFeatureImportanceValues(randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE))
             .build();
@@ -88,16 +88,16 @@ public void testConstructor_GivenEtaIsGreaterThanOne() {
 
     public void testConstructor_GivenMaximumNumberTreesIsZero() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> BoostedTreeParams.builder().setMaxTrees(0).build());
+            () -> BoostedTreeParams.builder().setMaximumNumberTrees(0).build());
 
-        assertThat(e.getMessage(), equalTo("[max_trees] must be an integer in [1, 2000]"));
+        assertThat(e.getMessage(), equalTo("[maximum_number_trees] must be an integer in [1, 2000]"));
     }
 
     public void testConstructor_GivenMaximumNumberTreesIsGreaterThan2k() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> BoostedTreeParams.builder().setMaxTrees(2001).build());
+            () -> BoostedTreeParams.builder().setMaximumNumberTrees(2001).build());
 
-        assertThat(e.getMessage(), equalTo("[max_trees] must be an integer in [1, 2000]"));
+        assertThat(e.getMessage(), equalTo("[maximum_number_trees] must be an integer in [1, 2000]"));
     }
 
     public void testConstructor_GivenFeatureBagFractionIsLessThanZero() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index 7429fe08d180d..0898a3fb476e8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -54,20 +54,17 @@ public static Classification createRandom() {
         String dependentVariableName = randomAlphaOfLength(10);
         BoostedTreeParams boostedTreeParams = BoostedTreeParamsTests.createRandom();
         String predictionFieldName = randomBoolean() ? null : randomAlphaOfLength(10);
-        Classification.ClassAssignmentObjective classAssignmentObjective = randomBoolean() ?
-            null : randomFrom(Classification.ClassAssignmentObjective.values());
         Integer numTopClasses = randomBoolean() ? null : randomIntBetween(0, 1000);
         Double trainingPercent = randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true);
         Long randomizeSeed = randomBoolean() ? null : randomLong();
-        return new Classification(dependentVariableName, boostedTreeParams, predictionFieldName, classAssignmentObjective,
-            numTopClasses, trainingPercent, randomizeSeed);
+        return new Classification(dependentVariableName, boostedTreeParams, predictionFieldName, numTopClasses, trainingPercent,
+            randomizeSeed);
     }
 
     public static Classification mutateForVersion(Classification instance, Version version) {
         return new Classification(instance.getDependentVariable(),
             BoostedTreeParamsTests.mutateForVersion(instance.getBoostedTreeParams(), version),
             instance.getPredictionFieldName(),
-            version.onOrAfter(Version.V_8_0_0) ? instance.getClassAssignmentObjective() : null,
             instance.getNumTopClasses(),
             instance.getTrainingPercent(),
             instance.getRandomizeSeed());
@@ -83,14 +80,12 @@ protected void assertOnBWCObject(Classification bwcSerializedObject, Classificat
         Classification newBwc = new Classification(bwcSerializedObject.getDependentVariable(),
             bwcSerializedObject.getBoostedTreeParams(),
             bwcSerializedObject.getPredictionFieldName(),
-            bwcSerializedObject.getClassAssignmentObjective(),
             bwcSerializedObject.getNumTopClasses(),
             bwcSerializedObject.getTrainingPercent(),
             42L);
         Classification newInstance = new Classification(testInstance.getDependentVariable(),
             testInstance.getBoostedTreeParams(),
             testInstance.getPredictionFieldName(),
-            testInstance.getClassAssignmentObjective(),
             testInstance.getNumTopClasses(),
             testInstance.getTrainingPercent(),
             42L);
@@ -104,129 +99,104 @@ protected Writeable.Reader<Classification> instanceReader() {
 
     public void testConstructor_GivenTrainingPercentIsLessThanOne() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 0.999, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 0.999, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[training_percent] must be a double in [1, 100]"));
     }
 
     public void testConstructor_GivenTrainingPercentIsGreaterThan100() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 100.0001, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 100.0001, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[training_percent] must be a double in [1, 100]"));
     }
 
     public void testConstructor_GivenNumTopClassesIsLessThanZero() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, -1, 1.0, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", -1, 1.0, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[num_top_classes] must be an integer in [0, 1000]"));
     }
 
     public void testConstructor_GivenNumTopClassesIsGreaterThan1000() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1001, 1.0, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 1001, 1.0, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[num_top_classes] must be an integer in [0, 1000]"));
     }
 
     public void testGetPredictionFieldName() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 50.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 50.0, randomLong());
         assertThat(classification.getPredictionFieldName(), equalTo("result"));
 
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, null, null, 3, 50.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, null, 3, 50.0, randomLong());
         assertThat(classification.getPredictionFieldName(), equalTo("foo_prediction"));
     }
 
-    public void testClassAssignmentObjective() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result",
-            Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY, 7, 1.0, randomLong());
-        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY));
-
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result",
-        Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL, 7, 1.0, randomLong());
-        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL));
-
-        // class_assignment_objective == null, default applied
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 7, 1.0, randomLong());
-        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL));
-    }
-
     public void testGetNumTopClasses() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 7, 1.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 7, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(7));
 
         // Boundary condition: num_top_classes == 0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 0, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 0, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(0));
 
         // Boundary condition: num_top_classes == 1000
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1000, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 1000, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(1000));
 
         // num_top_classes == null, default applied
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, null, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(2));
     }
 
     public void testGetTrainingPercent() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 50.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 50.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(50.0));
 
         // Boundary condition: training_percent == 1.0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 1.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(1.0));
 
         // Boundary condition: training_percent == 100.0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 100.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 100.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(100.0));
 
         // training_percent == null, default applied
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, null, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, null, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(100.0));
     }
 
     public void testGetParams() {
-        DataFrameAnalysis.FieldInfo fieldInfo = new TestFieldInfo(
+        Map<String, Set<String>> extractedFields =
             Map.of(
                 "foo", Set.of(BooleanFieldMapper.CONTENT_TYPE),
                 "bar", Set.of(NumberFieldMapper.NumberType.LONG.typeName()),
-                "baz", Set.of(KeywordFieldMapper.CONTENT_TYPE)),
-            Map.of(
-                "foo", 10L,
-                "bar", 20L,
-                "baz", 30L)
-        );
+                "baz", Set.of(KeywordFieldMapper.CONTENT_TYPE));
         assertThat(
-            new Classification("foo").getParams(fieldInfo),
+            new Classification("foo").getParams(extractedFields),
             equalTo(
                 Map.of(
                     "dependent_variable", "foo",
-                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "foo_prediction",
-                    "prediction_field_type", "bool",
-                    "num_classes", 10L)));
+                    "prediction_field_type", "bool")));
         assertThat(
-            new Classification("bar").getParams(fieldInfo),
+            new Classification("bar").getParams(extractedFields),
             equalTo(
                 Map.of(
                     "dependent_variable", "bar",
-                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "bar_prediction",
-                    "prediction_field_type", "int",
-                    "num_classes", 20L)));
+                    "prediction_field_type", "int")));
         assertThat(
-            new Classification("baz").getParams(fieldInfo),
+            new Classification("baz").getParams(extractedFields),
             equalTo(
                 Map.of(
                     "dependent_variable", "baz",
-                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "baz_prediction",
-                    "prediction_field_type", "string",
-                    "num_classes", 30L)));
+                    "prediction_field_type", "string")));
     }
 
     public void testRequiredFieldsIsNonEmpty() {
@@ -240,7 +210,7 @@ public void testFieldCardinalityLimitsIsNonEmpty() {
         assertThat(constraints.size(), equalTo(1));
         assertThat(constraints.get(0).getField(), equalTo(classification.getDependentVariable()));
         assertThat(constraints.get(0).getLowerBound(), equalTo(2L));
-        assertThat(constraints.get(0).getUpperBound(), equalTo(30L));
+        assertThat(constraints.get(0).getUpperBound(), equalTo(2L));
     }
 
     public void testGetExplicitlyMappedFields() {
@@ -339,25 +309,4 @@ public void testExtractJobIdFromStateDoc() {
     protected Classification mutateInstanceForVersion(Classification instance, Version version) {
         return mutateForVersion(instance, version);
     }
-
-    private static class TestFieldInfo implements DataFrameAnalysis.FieldInfo {
-
-        private final Map<String, Set<String>> fieldTypes;
-        private final Map<String, Long> fieldCardinalities;
-
-        private TestFieldInfo(Map<String, Set<String>> fieldTypes, Map<String, Long> fieldCardinalities) {
-            this.fieldTypes = fieldTypes;
-            this.fieldCardinalities = fieldCardinalities;
-        }
-
-        @Override
-        public Set<String> getTypes(String field) {
-            return fieldTypes.get(field);
-        }
-
-        @Override
-        public Long getCardinality(String field) {
-            return fieldCardinalities.get(field);
-        }
-    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java
deleted file mode 100644
index 1efe45fa930e4..0000000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml.dataframe.evaluation;
-
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class EvaluationParametersTests extends ESTestCase {
-
-    public void testConstructorAndGetters() {
-        EvaluationParameters params = new EvaluationParameters(17);
-        assertThat(params.getMaxBuckets(), equalTo(17));
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
index 1f755a93aad4a..16609dbdddcb0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
@@ -10,28 +10,25 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.PerClassResult;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.Result;
 
 import java.io.IOException;
 import java.util.List;
 
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockCardinality;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFilters;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFiltersBucket;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockSingleValue;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTermsBucket;
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
 public class AccuracyTests extends AbstractSerializingTestCase<Accuracy> {
 
-    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
-
     @Override
     protected Accuracy doParseInstance(XContentParser parser) throws IOException {
         return Accuracy.fromXContent(parser);
@@ -64,7 +61,6 @@ public void testProcess() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 100L),
-            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 1000L),
             mockFilters(
                 "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -80,12 +76,13 @@ public void testProcess() {
                         new Aggregations(List.of(mockFilters(
                             "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
                             List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L)))))))),
+            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 1000L),
             mockSingleValue(Accuracy.OVERALL_ACCURACY_AGG_NAME, 0.5)));
 
         Accuracy accuracy = new Accuracy();
         accuracy.process(aggs);
 
-        assertThat(accuracy.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
+        assertThat(accuracy.aggs("act", "pred"), isTuple(empty(), empty()));
 
         Result result = accuracy.getResult().get();
         assertThat(result.getMetricName(), equalTo(Accuracy.NAME.getPreferredName()));
@@ -106,7 +103,6 @@ public void testProcess_GivenCardinalityTooHigh() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 100L),
-            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 1001L),
             mockFilters(
                 "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -122,10 +118,11 @@ public void testProcess_GivenCardinalityTooHigh() {
                         new Aggregations(List.of(mockFilters(
                             "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
                             List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L)))))))),
+            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 1001L),
             mockSingleValue(Accuracy.OVERALL_ACCURACY_AGG_NAME, 0.5)));
 
         Accuracy accuracy = new Accuracy();
-        accuracy.aggs(EVALUATION_PARAMETERS, "foo", "bar");
+        accuracy.aggs("foo", "bar");
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> accuracy.process(aggs));
         assertThat(e.getMessage(), containsString("Cardinality of field [foo] is too high"));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
index bad673e28d7c9..f205974809961 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 
 import java.io.IOException;
@@ -44,8 +43,6 @@
 
 public class ClassificationTests extends AbstractSerializingTestCase<Classification> {
 
-    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
-
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(MlEvaluationNamedXContentProvider.getNamedWriteables());
@@ -103,7 +100,7 @@ public void testBuildSearch() {
 
         Classification evaluation = new Classification("act", "pred", Arrays.asList(new MulticlassConfusionMatrix()));
 
-        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(EVALUATION_PARAMETERS, userProvidedQuery);
+        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(userProvidedQuery);
         assertThat(searchSourceBuilder.query(), equalTo(expectedSearchQuery));
         assertThat(searchSourceBuilder.aggregations().count(), greaterThan(0));
     }
@@ -199,9 +196,7 @@ public String getWriteableName() {
         }
 
         @Override
-        public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
-                                                                                      String actualField,
-                                                                                      String predictedField) {
+        public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
             return Tuple.tuple(List.of(), List.of());
         }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
index 642bb9d915b17..98697d3536f32 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix.ActualClass;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix.PredictedClass;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix.Result;
@@ -22,20 +21,18 @@
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty;
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockCardinality;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFilters;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFiltersBucket;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTermsBucket;
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
 public class MulticlassConfusionMatrixTests extends AbstractSerializingTestCase<MulticlassConfusionMatrix> {
 
-    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
-
     @Override
     protected MulticlassConfusionMatrix doParseInstance(XContentParser parser) throws IOException {
         return MulticlassConfusionMatrix.fromXContent(parser);
@@ -81,12 +78,12 @@ public void testConstructor_SizeValidationFailures() {
 
     public void testAggs() {
         MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix();
-        Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs = confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred");
+        Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs = confusionMatrix.aggs("act", "pred");
         assertThat(aggs, isTuple(not(empty()), empty()));
         assertThat(confusionMatrix.getResult(), isEmpty());
     }
 
-    public void testProcess() {
+    public void testEvaluate() {
         Aggregations aggs = new Aggregations(List.of(
             mockTerms(
                 MulticlassConfusionMatrix.STEP_1_AGGREGATE_BY_ACTUAL_CLASS,
@@ -94,7 +91,6 @@ public void testProcess() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 0L),
-            mockCardinality(MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 2L),
             mockFilters(
                 MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -109,13 +105,13 @@ public void testProcess() {
                         70,
                         new Aggregations(List.of(mockFilters(
                             MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L))))))))
-        ));
+                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L)))))))),
+            mockCardinality(MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 2L)));
 
         MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix(2, null);
         confusionMatrix.process(aggs);
 
-        assertThat(confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
+        assertThat(confusionMatrix.aggs("act", "pred"), isTuple(empty(), empty()));
         Result result = confusionMatrix.getResult().get();
         assertThat(result.getMetricName(), equalTo(MulticlassConfusionMatrix.NAME.getPreferredName()));
         assertThat(
@@ -127,7 +123,7 @@ public void testProcess() {
         assertThat(result.getOtherActualClassCount(), equalTo(0L));
     }
 
-    public void testProcess_OtherClassesCountGreaterThanZero() {
+    public void testEvaluate_OtherClassesCountGreaterThanZero() {
         Aggregations aggs = new Aggregations(List.of(
             mockTerms(
                 MulticlassConfusionMatrix.STEP_1_AGGREGATE_BY_ACTUAL_CLASS,
@@ -135,7 +131,6 @@ public void testProcess_OtherClassesCountGreaterThanZero() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 100L),
-            mockCardinality(MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 5L),
             mockFilters(
                 MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -150,13 +145,13 @@ public void testProcess_OtherClassesCountGreaterThanZero() {
                         85,
                         new Aggregations(List.of(mockFilters(
                             MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 15L))))))))
-        ));
+                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 15L)))))))),
+            mockCardinality(MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 5L)));
 
         MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix(2, null);
         confusionMatrix.process(aggs);
 
-        assertThat(confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
+        assertThat(confusionMatrix.aggs("act", "pred"), isTuple(empty(), empty()));
         Result result = confusionMatrix.getResult().get();
         assertThat(result.getMetricName(), equalTo(MulticlassConfusionMatrix.NAME.getPreferredName()));
         assertThat(
@@ -167,106 +162,4 @@ public void testProcess_OtherClassesCountGreaterThanZero() {
                     new ActualClass("cat", 85, List.of(new PredictedClass("cat", 30L), new PredictedClass("dog", 40L)), 15))));
         assertThat(result.getOtherActualClassCount(), equalTo(3L));
     }
-
-    public void testProcess_MoreThanTwoStepsNeeded() {
-        Aggregations aggsStep1 = new Aggregations(List.of(
-            mockTerms(
-                MulticlassConfusionMatrix.STEP_1_AGGREGATE_BY_ACTUAL_CLASS,
-                List.of(
-                    mockTermsBucket("ant", new Aggregations(List.of())),
-                    mockTermsBucket("cat", new Aggregations(List.of())),
-                    mockTermsBucket("dog", new Aggregations(List.of())),
-                    mockTermsBucket("fox", new Aggregations(List.of()))),
-                0L),
-            mockCardinality(MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 2L)
-        ));
-        Aggregations aggsStep2 = new Aggregations(List.of(
-            mockFilters(
-                MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
-                List.of(
-                    mockFiltersBucket(
-                        "ant",
-                        46,
-                        new Aggregations(List.of(mockFilters(
-                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(
-                                mockFiltersBucket("ant", 10L),
-                                mockFiltersBucket("cat", 11L),
-                                mockFiltersBucket("dog", 12L),
-                                mockFiltersBucket("fox", 13L),
-                                mockFiltersBucket("_other_", 0L)))))),
-                    mockFiltersBucket(
-                        "cat",
-                        86,
-                        new Aggregations(List.of(mockFilters(
-                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(
-                                mockFiltersBucket("ant", 20L),
-                                mockFiltersBucket("cat", 21L),
-                                mockFiltersBucket("dog", 22L),
-                                mockFiltersBucket("fox", 23L),
-                                mockFiltersBucket("_other_", 0L))))))))
-        ));
-        Aggregations aggsStep3 = new Aggregations(List.of(
-            mockFilters(
-                MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
-                List.of(
-                    mockFiltersBucket(
-                        "dog",
-                        126,
-                        new Aggregations(List.of(mockFilters(
-                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(
-                                mockFiltersBucket("ant", 30L),
-                                mockFiltersBucket("cat", 31L),
-                                mockFiltersBucket("dog", 32L),
-                                mockFiltersBucket("fox", 33L),
-                                mockFiltersBucket("_other_", 0L)))))),
-                    mockFiltersBucket(
-                        "fox",
-                        166,
-                        new Aggregations(List.of(mockFilters(
-                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(
-                                mockFiltersBucket("ant", 40L),
-                                mockFiltersBucket("cat", 41L),
-                                mockFiltersBucket("dog", 42L),
-                                mockFiltersBucket("fox", 43L),
-                                mockFiltersBucket("_other_", 0L))))))))
-        ));
-
-        MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix(4, null);
-        confusionMatrix.process(aggsStep1);
-        confusionMatrix.process(aggsStep2);
-        confusionMatrix.process(aggsStep3);
-
-        assertThat(confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
-        Result result = confusionMatrix.getResult().get();
-        assertThat(result.getMetricName(), equalTo(MulticlassConfusionMatrix.NAME.getPreferredName()));
-        assertThat(
-            result.getConfusionMatrix(),
-            equalTo(
-                List.of(
-                    new ActualClass("ant", 46, List.of(
-                        new PredictedClass("ant", 10L),
-                        new PredictedClass("cat", 11L),
-                        new PredictedClass("dog", 12L),
-                        new PredictedClass("fox", 13L)), 0),
-                    new ActualClass("cat", 86, List.of(
-                        new PredictedClass("ant", 20L),
-                        new PredictedClass("cat", 21L),
-                        new PredictedClass("dog", 22L),
-                        new PredictedClass("fox", 23L)), 0),
-                    new ActualClass("dog", 126, List.of(
-                        new PredictedClass("ant", 30L),
-                        new PredictedClass("cat", 31L),
-                        new PredictedClass("dog", 32L),
-                        new PredictedClass("fox", 33L)), 0),
-                    new ActualClass("fox", 166, List.of(
-                        new PredictedClass("ant", 40L),
-                        new PredictedClass("cat", 41L),
-                        new PredictedClass("dog", 42L),
-                        new PredictedClass("fox", 43L)), 0))));
-        assertThat(result.getOtherActualClassCount(), equalTo(0L));
-    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
deleted file mode 100644
index 5a397bae366d7..0000000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification;
-
-import org.elasticsearch.script.Script;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class PainlessScriptsTests extends ESTestCase {
-
-    public void testBuildIsEqualScript() {
-        Script script = PainlessScripts.buildIsEqualScript("act", "pred");
-        assertThat(script.getIdOrCode(), equalTo("String.valueOf(doc['act'].value).equals(String.valueOf(doc['pred'].value))"));
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
index 5e49412e3238b..362b81e9ae981 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -18,18 +17,16 @@
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty;
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFilters;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockSingleValue;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
 public class PrecisionTests extends AbstractSerializingTestCase<Precision> {
 
-    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
-
     @Override
     protected Precision doParseInstance(XContentParser parser) throws IOException {
         return Precision.fromXContent(parser);
@@ -65,7 +62,7 @@ public void testProcess() {
         Precision precision = new Precision();
         precision.process(aggs);
 
-        assertThat(precision.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
+        assertThat(precision.aggs("act", "pred"), isTuple(empty(), empty()));
         assertThat(precision.getResult().get(), equalTo(new Precision.Result(List.of(), 0.8123)));
     }
 
@@ -115,7 +112,7 @@ public void testProcess_GivenCardinalityTooHigh() {
         Aggregations aggs =
             new Aggregations(Collections.singletonList(mockTerms(Precision.ACTUAL_CLASSES_NAMES_AGG_NAME, Collections.emptyList(), 1)));
         Precision precision = new Precision();
-        precision.aggs(EVALUATION_PARAMETERS, "foo", "bar");
+        precision.aggs("foo", "bar");
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> precision.process(aggs));
         assertThat(e.getMessage(), containsString("Cardinality of field [foo] is too high"));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
index 721942203f779..be7d30411c931 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -18,17 +17,15 @@
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty;
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockSingleValue;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
 public class RecallTests extends AbstractSerializingTestCase<Recall> {
 
-    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
-
     @Override
     protected Recall doParseInstance(XContentParser parser) throws IOException {
         return Recall.fromXContent(parser);
@@ -63,7 +60,7 @@ public void testProcess() {
         Recall recall = new Recall();
         recall.process(aggs);
 
-        assertThat(recall.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
+        assertThat(recall.aggs("act", "pred"), isTuple(empty(), empty()));
         assertThat(recall.getResult().get(), equalTo(new Recall.Result(List.of(), 0.8123)));
     }
 
@@ -114,7 +111,7 @@ public void testProcess_GivenCardinalityTooHigh() {
             mockTerms(Recall.BY_ACTUAL_CLASS_AGG_NAME, Collections.emptyList(), 1),
             mockSingleValue(Recall.AVG_RECALL_AGG_NAME, 0.8123)));
         Recall recall = new Recall();
-        recall.aggs(EVALUATION_PARAMETERS, "foo", "bar");
+        recall.aggs("foo", "bar");
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> recall.process(aggs));
         assertThat(e.getMessage(), containsString("Cardinality of field [foo] is too high"));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
index 26dff097b1b32..96ba97ecc9348 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 
 import java.io.IOException;
@@ -29,8 +28,6 @@
 
 public class RegressionTests extends AbstractSerializingTestCase<Regression> {
 
-    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
-
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(MlEvaluationNamedXContentProvider.getNamedWriteables());
@@ -88,7 +85,7 @@ public void testBuildSearch() {
 
         Regression evaluation = new Regression("act", "pred", Arrays.asList(new MeanSquaredError()));
 
-        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(EVALUATION_PARAMETERS, userProvidedQuery);
+        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(userProvidedQuery);
         assertThat(searchSourceBuilder.query(), equalTo(expectedSearchQuery));
         assertThat(searchSourceBuilder.aggregations().count(), greaterThan(0));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
index 2a9645e094291..28e0a045b190d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 
 import java.io.IOException;
@@ -29,8 +28,6 @@
 
 public class BinarySoftClassificationTests extends AbstractSerializingTestCase<BinarySoftClassification> {
 
-    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
-
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(MlEvaluationNamedXContentProvider.getNamedWriteables());
@@ -101,7 +98,7 @@ public void testBuildSearch() {
 
         BinarySoftClassification evaluation = new BinarySoftClassification("act", "prob", Arrays.asList(new Precision(Arrays.asList(0.7))));
 
-        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(EVALUATION_PARAMETERS, userProvidedQuery);
+        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(userProvidedQuery);
         assertThat(searchSourceBuilder.query(), equalTo(expectedSearchQuery));
         assertThat(searchSourceBuilder.aggregations().count(), greaterThan(0));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
index a41f0564fb967..11e66e69c0f75 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
@@ -34,11 +34,9 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
 import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_INTERNAL_STORAGE;
@@ -139,11 +137,7 @@ public void testToXContentWithParams() throws IOException {
             TrainedModelInputTests.createRandomInput(),
             randomNonNegativeLong(),
             randomNonNegativeLong(),
-            "platinum",
-            randomBoolean() ? null :
-                Stream.generate(() -> randomAlphaOfLength(10))
-                    .limit(randomIntBetween(1, 10))
-                    .collect(Collectors.toMap(Function.identity(), (k) -> randomAlphaOfLength(10))));
+            "platinum");
 
         BytesReference reference = XContentHelper.toXContent(config, XContentType.JSON, ToXContent.EMPTY_PARAMS, false);
         assertThat(reference.utf8ToString(), containsString("\"compressed_definition\""));
@@ -178,11 +172,7 @@ public void testParseWithBothDefinitionAndCompressedSupplied() throws IOExceptio
             TrainedModelInputTests.createRandomInput(),
             randomNonNegativeLong(),
             randomNonNegativeLong(),
-            "platinum",
-            randomBoolean() ? null :
-                Stream.generate(() -> randomAlphaOfLength(10))
-                    .limit(randomIntBetween(1, 10))
-                    .collect(Collectors.toMap(Function.identity(), (k) -> randomAlphaOfLength(10))));
+            "platinum");
 
         BytesReference reference = XContentHelper.toXContent(config, XContentType.JSON, ToXContent.EMPTY_PARAMS, false);
         Map<String, Object> objectMap = XContentHelper.convertToMap(reference, true, XContentType.JSON).v2();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
index 850d197aa6d8b..80a55394a4003 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
@@ -47,7 +47,6 @@
 
 import static java.util.stream.Collectors.toMap;
 import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doAnswer;
@@ -68,7 +67,7 @@ public class MlIndexAndAliasTests extends ESTestCase {
     private IndicesAdminClient indicesAdminClient;
     private AdminClient adminClient;
     private Client client;
-    private ActionListener<Boolean> listener;
+    private ActionListener<Boolean> finalListener;
 
     private ArgumentCaptor<CreateIndexRequest> createRequestCaptor;
     private ArgumentCaptor<IndicesAliasesRequest> aliasesRequestCaptor;
@@ -92,7 +91,7 @@ public void setUpMocks() {
         when(client.threadPool()).thenReturn(threadPool);
         when(client.admin()).thenReturn(adminClient);
 
-        listener = mock(ActionListener.class);
+        finalListener = mock(ActionListener.class);
 
         createRequestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
         aliasesRequestCaptor = ArgumentCaptor.forClass(IndicesAliasesRequest.class);
@@ -100,17 +99,17 @@ public void setUpMocks() {
 
     @After
     public void verifyNoMoreInteractionsWithMocks() {
-        verifyNoMoreInteractions(indicesAdminClient, listener);
+        verifyNoMoreInteractions(indicesAdminClient, finalListener);
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_CleanState() {
         ClusterState clusterState = createClusterState(Collections.emptyMap());
         createIndexAndAliasIfNecessary(clusterState);
 
-        InOrder inOrder = inOrder(indicesAdminClient, listener);
+        InOrder inOrder = inOrder(indicesAdminClient, finalListener);
         inOrder.verify(indicesAdminClient).prepareCreate(FIRST_CONCRETE_INDEX);
         inOrder.verify(indicesAdminClient).create(createRequestCaptor.capture(), any());
-        inOrder.verify(listener).onResponse(true);
+        inOrder.verify(finalListener).onResponse(true);
 
         CreateIndexRequest createRequest = createRequestCaptor.getValue();
         assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
@@ -121,7 +120,11 @@ private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexN
         ClusterState clusterState = createClusterState(Collections.singletonMap(indexName, createIndexMetaDataWithAlias(indexName)));
         createIndexAndAliasIfNecessary(clusterState);
 
-        verify(listener).onResponse(false);
+        verify(finalListener).onResponse(false);
+    }
+
+    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtLegacyIndex() {
+        assertNoClientInteractionsWhenWriteAliasAlreadyExists(LEGACY_INDEX_WITHOUT_SUFFIX);
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtInitialStateIndex() {
@@ -129,48 +132,23 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPo
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtSubsequentStateIndex() {
-        assertNoClientInteractionsWhenWriteAliasAlreadyExists("test-000007");
+        assertNoClientInteractionsWhenWriteAliasAlreadyExists(".ml-state-000007");
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtDummyIndex() {
         assertNoClientInteractionsWhenWriteAliasAlreadyExists("dummy-index");
     }
 
-    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtLegacyStateIndex() {
-        ClusterState clusterState =
-            createClusterState(
-                Collections.singletonMap(LEGACY_INDEX_WITHOUT_SUFFIX, createIndexMetaDataWithAlias(LEGACY_INDEX_WITHOUT_SUFFIX)));
-        createIndexAndAliasIfNecessary(clusterState);
-
-        InOrder inOrder = inOrder(indicesAdminClient, listener);
-        inOrder.verify(indicesAdminClient).prepareCreate(FIRST_CONCRETE_INDEX);
-        inOrder.verify(indicesAdminClient).create(createRequestCaptor.capture(), any());
-        inOrder.verify(indicesAdminClient).prepareAliases();
-        inOrder.verify(indicesAdminClient).aliases(aliasesRequestCaptor.capture(), any());
-        inOrder.verify(listener).onResponse(true);
-
-        CreateIndexRequest createRequest = createRequestCaptor.getValue();
-        assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
-        assertThat(createRequest.aliases(), empty());
-
-        IndicesAliasesRequest indicesAliasesRequest = aliasesRequestCaptor.getValue();
-        assertThat(
-            indicesAliasesRequest.getAliasActions(),
-            contains(
-                AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX),
-                AliasActions.remove().alias(TEST_INDEX_ALIAS).index(LEGACY_INDEX_WITHOUT_SUFFIX)));
-    }
-
     private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> existingIndexNames, String expectedWriteIndexName) {
         ClusterState clusterState =
             createClusterState(
                 existingIndexNames.stream().collect(toMap(Function.identity(), MlIndexAndAliasTests::createIndexMetaData)));
         createIndexAndAliasIfNecessary(clusterState);
 
-        InOrder inOrder = inOrder(indicesAdminClient, listener);
+        InOrder inOrder = inOrder(indicesAdminClient, finalListener);
         inOrder.verify(indicesAdminClient).prepareAliases();
         inOrder.verify(indicesAdminClient).aliases(aliasesRequestCaptor.capture(), any());
-        inOrder.verify(listener).onResponse(true);
+        inOrder.verify(finalListener).onResponse(true);
 
         IndicesAliasesRequest indicesAliasesRequest = aliasesRequestCaptor.getValue();
         assertThat(
@@ -178,6 +156,11 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> e
             contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName)));
     }
 
+    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLegacyIndexExists() {
+        assertMlStateWriteAliasAddedToMostRecentMlStateIndex(
+            Arrays.asList(LEGACY_INDEX_WITHOUT_SUFFIX), LEGACY_INDEX_WITHOUT_SUFFIX);
+    }
+
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() {
         assertMlStateWriteAliasAddedToMostRecentMlStateIndex(
             Arrays.asList(FIRST_CONCRETE_INDEX), FIRST_CONCRETE_INDEX);
@@ -193,21 +176,6 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButBot
             Arrays.asList(LEGACY_INDEX_WITHOUT_SUFFIX, "test-000003", "test-000040", "test-000500"), "test-000500");
     }
 
-    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLegacyStateIndexExists() {
-        ClusterState clusterState =
-            createClusterState(Collections.singletonMap(LEGACY_INDEX_WITHOUT_SUFFIX, createIndexMetaData(LEGACY_INDEX_WITHOUT_SUFFIX)));
-        createIndexAndAliasIfNecessary(clusterState);
-
-        InOrder inOrder = inOrder(indicesAdminClient, listener);
-        inOrder.verify(indicesAdminClient).prepareCreate(FIRST_CONCRETE_INDEX);
-        inOrder.verify(indicesAdminClient).create(createRequestCaptor.capture(), any());
-        inOrder.verify(listener).onResponse(true);
-
-        CreateIndexRequest createRequest = createRequestCaptor.getValue();
-        assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
-        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
-    }
-
     public void testIndexNameComparator() {
         Comparator<String> comparator = MlIndexAndAlias.INDEX_NAME_COMPARATOR;
         assertThat(
@@ -234,8 +202,8 @@ public void testIndexNameComparator() {
     }
 
     private void createIndexAndAliasIfNecessary(ClusterState clusterState) {
-        MlIndexAndAlias.createIndexAndAliasIfNecessary(
-            client, clusterState, new IndexNameExpressionResolver(), TEST_INDEX_PREFIX, TEST_INDEX_ALIAS, listener);
+        MlIndexAndAlias.createIndexAndAliasIfNecessary(client, clusterState, new IndexNameExpressionResolver(),
+            TEST_INDEX_PREFIX, TEST_INDEX_ALIAS, finalListener);
     }
 
     @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
deleted file mode 100644
index c5087622404ab..0000000000000
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.deprecation;
-
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
-
-import java.util.Locale;
-
-public class NodeDeprecationChecks {
-
-    static DeprecationIssue checkRemovedSetting(final Settings settings, final Setting<?> removedSetting, final String url) {
-        if (removedSetting.exists(settings) == false) {
-            return null;
-        }
-        final String removedSettingKey = removedSetting.getKey();
-        final String value = removedSetting.get(settings).toString();
-        final String message =
-            String.format(Locale.ROOT, "setting [%s] is deprecated and will be removed in the next major version", removedSettingKey);
-        final String details =
-            String.format(Locale.ROOT, "the setting [%s] is currently set to [%s], remove this setting", removedSettingKey, value);
-        return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details);
-    }
-
-}
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
deleted file mode 100644
index 00c49aedbe223..0000000000000
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.deprecation;
-
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
-
-public class NodeDeprecationChecksTests extends ESTestCase {
-
-    public void testRemovedSettingNotSet() {
-        final Settings settings = Settings.EMPTY;
-        final Setting<?> removedSetting = Setting.simpleString("node.removed_setting");
-        final DeprecationIssue issue =
-            NodeDeprecationChecks.checkRemovedSetting(settings, removedSetting, "http://removed-setting.example.com");
-        assertThat(issue, nullValue());
-    }
-
-    public void testRemovedSetting() {
-        final Settings settings = Settings.builder().put("node.removed_setting", "value").build();
-        final Setting<?> removedSetting = Setting.simpleString("node.removed_setting");
-        final DeprecationIssue issue =
-            NodeDeprecationChecks.checkRemovedSetting(settings, removedSetting, "https://removed-setting.example.com");
-        assertThat(issue, not(nullValue()));
-        assertThat(issue.getLevel(), equalTo(DeprecationIssue.Level.CRITICAL));
-        assertThat(
-            issue.getMessage(),
-            equalTo("setting [node.removed_setting] is deprecated and will be removed in the next major version"));
-        assertThat(
-            issue.getDetails(),
-            equalTo("the setting [node.removed_setting] is currently set to [value], remove this setting"));
-        assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com"));
-    }
-
-}
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java
index 1b6306fcb2a5c..f857232c7e910 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java
@@ -16,28 +16,53 @@
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.util.ArrayList;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
 public abstract class CommonEqlRestTestCase extends ESRestTestCase {
 
-    private static final String defaultValidationIndexName = "eql_search_validation_test";
+    static class SearchTestConfiguration {
+        final String input;
+        final int expectedStatus;
+        final String expectedMessage;
+
+        SearchTestConfiguration(String input, int status, String msg) {
+            this.input = input;
+            this.expectedStatus = status;
+            this.expectedMessage = msg;
+        }
+    }
+
+    public static final String defaultValidationIndexName = "eql_search_validation_test";
     private static final String validQuery = "process where user = 'SYSTEM'";
 
-    private static final String[][] testBadRequests = {
-            {null, "request body or source parameter is required"},
-            {"{}", "query is null or empty"},
-            {"{\"query\": \"\"}", "query is null or empty"},
-            {"{\"query\": \"" + validQuery + "\", \"timestamp_field\": \"\"}", "timestamp field is null or empty"},
-            {"{\"query\": \"" + validQuery + "\", \"event_category_field\": \"\"}", "event category field is null or empty"},
-            {"{\"query\": \"" + validQuery + "\", \"implicit_join_key_field\": \"\"}", "implicit join key field is null or empty"},
-            {"{\"query\": \"" + validQuery + "\", \"size\": 0}", "size must be greater than 0"},
-            {"{\"query\": \"" + validQuery + "\", \"size\": -1}", "size must be greater than 0"},
-            {"{\"query\": \"" + validQuery + "\", \"search_after\": null}", "search_after doesn't support values of type: VALUE_NULL"},
-            {"{\"query\": \"" + validQuery + "\", \"search_after\": []}", "must contains at least one value"},
-            {"{\"query\": \"" + validQuery + "\", \"filter\": null}", "filter doesn't support values of type: VALUE_NULL"},
-            {"{\"query\": \"" + validQuery + "\", \"filter\": {}}", "query malformed, empty clause found"}
-    };
+    public static final ArrayList<SearchTestConfiguration> searchValidationTests;
+    static {
+        searchValidationTests = new ArrayList<>();
+        searchValidationTests.add(new SearchTestConfiguration(null, 400, "request body or source parameter is required"));
+        searchValidationTests.add(new SearchTestConfiguration("{}", 400, "query is null or empty"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"\"}", 400, "query is null or empty"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"timestamp_field\": \"\"}",
+            400, "timestamp field is null or empty"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"event_category_field\": \"\"}",
+            400, "event category field is null or empty"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"implicit_join_key_field\": \"\"}",
+            400, "implicit join key field is null or empty"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"size\": 0}",
+            400, "size must be greater than 0"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"size\": -1}",
+            400, "size must be greater than 0"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"search_after\": null}",
+            400, "search_after doesn't support values of type: VALUE_NULL"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"search_after\": []}",
+            400, "must contains at least one value"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"filter\": null}",
+            400, "filter doesn't support values of type: VALUE_NULL"));
+        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"filter\": {}}",
+            400, "query malformed, empty clause found"));
+    }
 
     @BeforeClass
     public static void checkForSnapshot() {
@@ -54,19 +79,24 @@ public void cleanup() throws Exception {
         deleteIndex(defaultValidationIndexName);
     }
 
-    public void testBadRequests() throws Exception {
+    public void testSearchValidationFailures() throws Exception {
         final String contentType = "application/json";
-        for (String[] test : testBadRequests) {
+        for (SearchTestConfiguration config : searchValidationTests) {
             final String endpoint = "/" + defaultValidationIndexName + "/_eql/search";
             Request request = new Request("GET", endpoint);
-            request.setJsonEntity(test[0]);
+            request.setJsonEntity(config.input);
 
-            ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
-            Response response = e.getResponse();
+            Response response = null;
+            if (config.expectedStatus == 400) {
+                ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
+                response = e.getResponse();
+            } else {
+                response = client().performRequest(request);
+            }
 
             assertThat(response.getHeader("Content-Type"), containsString(contentType));
-            assertThat(EntityUtils.toString(response.getEntity()), containsString(test[1]));
-            assertThat(response.getStatusLine().getStatusCode(), is(400));
+            assertThat(EntityUtils.toString(response.getEntity()), containsString(config.expectedMessage));
+            assertThat(response.getStatusLine().getStatusCode(), is(config.expectedStatus));
         }
     }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
index aea8042c07ec0..979c622380072 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
@@ -8,8 +8,6 @@
 
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
-import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
-import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
@@ -45,7 +43,6 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
                 new BooleanLiteralsOnTheRight(),
                 // needs to occur before BinaryComparison combinations
                 new ReplaceWildcards(),
-                new ReplaceNullChecks(),
                 new PropagateEquals(),
                 new CombineBinaryComparisons(),
                 // prune/elimination
@@ -105,28 +102,4 @@ protected LogicalPlan rule(Filter filter) {
             });
         }
     }
-
-    private static class ReplaceNullChecks extends OptimizerRule<Filter> {
-
-        @Override
-        protected LogicalPlan rule(Filter filter) {
-
-            return filter.transformExpressionsUp(e -> {
-                // expr == null || expr != null
-                if (e instanceof Equals || e instanceof NotEquals) {
-                    BinaryComparison cmp = (BinaryComparison) e;
-
-                    if (cmp.right().foldable() && cmp.right().fold() == null) {
-                        if (e instanceof Equals) {
-                            e = new IsNull(e.source(), cmp.left());
-                        } else {
-                            e = new IsNotNull(e.source(), cmp.left());
-                        }
-                    }
-                }
-
-                return e;
-            });
-        }
-    }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
index cf313acd1a1b6..17f9f0da8c6f9 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
@@ -7,32 +7,33 @@
 package org.elasticsearch.xpack.eql.action;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.elasticsearch.Build;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.SearchHit;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
 
 public class EqlActionIT extends AbstractEqlIntegTestCase {
 
     static final String indexPrefix = "endgame";
     static final String testIndexName = indexPrefix + "-1.4.0";
-    protected static final String PARAM_FORMATTING = "%1$s.test -> %2$s";
+    protected static final String PARAM_FORMATTING = "%1$s.test";
+
 
     @BeforeClass
     public static void checkForSnapshot() {
@@ -40,29 +41,26 @@ public static void checkForSnapshot() {
     }
 
     @Before
-    @SuppressWarnings("unchecked")
     public void setUpData() throws Exception {
         // Insert test data
         ObjectMapper mapper = new ObjectMapper();
         BulkRequestBuilder bulkBuilder = client().prepareBulk();
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, EqlActionIT.class.getResourceAsStream("/test_data.json"))) {
-            List<Object> list = parser.list();
-            for (Object item : list) {
-                assertThat(item, instanceOf(HashMap.class));
-
-                HashMap<String, Object> entry = (HashMap<String, Object>) item;
-
-                // Adjust the structure of the document with additional event.category and @timestamp fields
-                // Add event.category field
-                HashMap<String, Object> objEvent = new HashMap<>();
-                objEvent.put("category", entry.get("event_type"));
-                entry.put("event", objEvent);
-
-                // Add @timestamp field
-                entry.put("@timestamp", entry.get("timestamp"));
-
-                bulkBuilder.add(new IndexRequest(testIndexName).source(entry, XContentType.JSON));
-            }
+        JsonNode rootNode = mapper.readTree(EqlActionIT.class.getResourceAsStream("/test_data.json"));
+        Iterator<JsonNode> entries = rootNode.elements();
+        while (entries.hasNext()) {
+            JsonNode entry = entries.next();
+
+            // Adjust the structure of the document with additional event.category and @timestamp fields
+            // Add event.category field
+            ObjectNode objEvent = ((ObjectNode)entry).putObject("event");
+            JsonNode objEventType = entry.get("event_type");
+            objEvent.put("category", objEventType.asText());
+
+            // Add @timestamp field
+            JsonNode objTimestamp = entry.get("timestamp");
+            ((ObjectNode)entry).put("@timestamp", objTimestamp.asLong());
+
+            bulkBuilder.add(new IndexRequest(testIndexName).source(entry.toString(), XContentType.JSON));
         }
         BulkResponse bulkResponse = bulkBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
         assertThat(bulkResponse.hasFailures() ? bulkResponse.buildFailureMessage() : "", bulkResponse.hasFailures(), equalTo(false));
@@ -110,24 +108,19 @@ public EqlActionIT(int num, EqlSpec spec) {
         this.spec = spec;
     }
 
-    private static long[] extractIds(List<SearchHit> events) {
+    public final void test() {
+        EqlSearchResponse response = new EqlSearchRequestBuilder(client(), EqlSearchAction.INSTANCE)
+            .indices(testIndexName).query(spec.query()).get();
+
+        List<SearchHit> events = response.hits().events();
+        assertNotNull(events);
+
         final int len = events.size();
         final long ids[] = new long[len];
-        for (int i = 0; i < len; i++) {
+        for (int i = 0; i < events.size(); i++) {
             ids[i] = ((Number) events.get(i).getSourceAsMap().get("serial_event_id")).longValue();
         }
-        return ids;
-    }
-
-    private void assertSpec(List<SearchHit> events) {
-        assertNotNull(events);
-        assertArrayEquals("unexpected result for spec: [" + spec.toString() + "]", spec.expectedEventIds(), extractIds(events));
-    }
-
-    public final void test() {
-        EqlSearchResponse response = new EqlSearchRequestBuilder(client(), EqlSearchAction.INSTANCE)
-                .indices(testIndexName).query(spec.query()).get();
-
-        assertSpec(response.hits().events());
+        final String msg = "unexpected result for spec: [" + spec.toString() + "]";
+        assertArrayEquals(msg, spec.expectedEventIds(), ids);
     }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
index bbd589331b8ad..6b53fec8caafb 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
@@ -15,8 +15,6 @@
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
-import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
-import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
@@ -26,8 +24,6 @@
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.ql.type.TypesTests;
 
-import java.util.Arrays;
-import java.util.List;
 import java.util.Map;
 
 public class OptimizerTests extends ESTestCase {
@@ -36,7 +32,6 @@ public class OptimizerTests extends ESTestCase {
     private static final String INDEX_NAME = "test";
     private EqlParser parser = new EqlParser();
     private IndexResolution index = loadIndexResolution("mapping-default.json");
-
     private static Map<String, EsField> loadEqlMapping(String name) {
         return TypesTests.loadMapping(name);
     }
@@ -56,54 +51,9 @@ private LogicalPlan accept(String eql) {
         return accept(index, eql);
     }
 
-    public void testIsNull() {
-        List<String> tests = Arrays.asList(
-            "foo where command_line == null",
-            "foo where null == command_line"
-        );
-
-        for (String q : tests) {
-            LogicalPlan plan = accept(q);
-            assertTrue(plan instanceof OrderBy);
-            plan = ((OrderBy) plan).child();
-            assertTrue(plan instanceof Filter);
-
-            Filter filter = (Filter) plan;
-            And condition = (And) filter.condition();
-            assertTrue(condition.right() instanceof IsNull);
-
-            IsNull check = (IsNull) condition.right();
-            assertEquals(((FieldAttribute) check.field()).name(), "command_line");
-        }
-    }
-    public void testIsNotNull() {
-        List<String> tests = Arrays.asList(
-            "foo where command_line != null",
-            "foo where null != command_line"
-        );
-
-        for (String q : tests) {
-            LogicalPlan plan = accept(q);
-            assertTrue(plan instanceof OrderBy);
-            plan = ((OrderBy) plan).child();
-            assertTrue(plan instanceof Filter);
-
-            Filter filter = (Filter) plan;
-            And condition = (And) filter.condition();
-            assertTrue(condition.right() instanceof IsNotNull);
-
-            IsNotNull check = (IsNotNull) condition.right();
-            assertEquals(((FieldAttribute) check.field()).name(), "command_line");
-        }
-    }
 
     public void testEqualsWildcard() {
-        List<String> tests = Arrays.asList(
-            "foo where command_line == '* bar *'",
-            "foo where '* bar *' == command_line"
-        );
-
-        for (String q : tests) {
+        for (String q : new String[]{"foo where command_line == '* bar *'", "foo where '* bar *' == command_line"}) {
             LogicalPlan plan = accept(q);
             assertTrue(plan instanceof OrderBy);
             plan = ((OrderBy) plan).child();
@@ -122,12 +72,7 @@ public void testEqualsWildcard() {
     }
 
     public void testNotEqualsWildcard() {
-        List<String> tests = Arrays.asList(
-            "foo where command_line != '* baz *'",
-            "foo where '* baz *' != command_line"
-        );
-
-        for (String q : tests) {
+        for (String q : new String[]{"foo where command_line != '* baz *'", "foo where '* baz *' != command_line"}) {
 
             LogicalPlan plan = accept(q);
             assertTrue(plan instanceof OrderBy);
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java
deleted file mode 100644
index 7c54e31db79be..0000000000000
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.eql.planner;
-
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.eql.analysis.Analyzer;
-import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
-import org.elasticsearch.xpack.eql.analysis.Verifier;
-import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
-import org.elasticsearch.xpack.eql.optimizer.Optimizer;
-import org.elasticsearch.xpack.eql.parser.EqlParser;
-import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.ql.index.EsIndex;
-import org.elasticsearch.xpack.ql.index.IndexResolution;
-
-import static org.elasticsearch.xpack.ql.type.TypesTests.loadMapping;
-
-public abstract class AbstractQueryFolderTestCase extends ESTestCase {
-    protected EqlParser parser = new EqlParser();
-    protected PreAnalyzer preAnalyzer = new PreAnalyzer();
-    protected Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier());
-    protected Optimizer optimizer = new Optimizer();
-    protected Planner planner = new Planner();
-
-    protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json")));
-
-    protected PhysicalPlan plan(IndexResolution resolution, String eql) {
-        return planner.plan(optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))));
-    }
-
-    protected PhysicalPlan plan(String eql) {
-        return plan(index, eql);
-    }
-}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
deleted file mode 100644
index 471ee4c56943a..0000000000000
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.eql.planner;
-
-import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
-
-public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
-    public void testPropertyEquationFilterUnsupported() {
-        QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
-                () -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)"));
-        String msg = e.getMessage();
-        assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg);
-    }
-
-    public void testPropertyEquationInClauseFilterUnsupported() {
-        QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
-                () -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")"));
-        String msg = e.getMessage();
-        assertEquals("Line 1:52: Comparisons against variables are not (currently) supported; offender [parent_process_name] in [==]", msg);
-    }
-}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
deleted file mode 100644
index acbe328a4740a..0000000000000
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.eql.planner;
-
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
-import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
-
-import java.util.Arrays;
-
-import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
-import static org.hamcrest.Matchers.containsString;
-
-public class QueryFolderOkTests extends AbstractQueryFolderTestCase {
-    private static Object[][] specs = {
-            {"basic", "process where true", null},
-            {"singleNumericFilterEquals", "process where serial_event_id = 1", "\"term\":{\"serial_event_id\":{\"value\":1"},
-            {"singleNumericFilterLess", "process where serial_event_id < 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":false"
-            },
-            {"singleNumericFilterLessSymmetry", "process where 4 > serial_event_id",
-                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":false"
-            },
-            {"singleNumericFilterLessEquals", "process where serial_event_id <= 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":true"
-            },
-            {"singleNumericFilterGreater", "process where serial_event_id > 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":4,\"to\":null,\"include_lower\":false,\"include_upper\":false"
-            },
-            {"singleNumericFilterGreaterEquals", "process where serial_event_id >= 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":4,\"to\":null,\"include_lower\":true,\"include_upper\":false"
-            },
-            {"mixedTypeFilter", "process where process_name == \"notepad.exe\" or (serial_event_id < 4.5 and serial_event_id >= 3.1)",
-                    new Object[]{
-                            "\"term\":{\"process_name\":{\"value\":\"notepad.exe\"",
-                            "\"range\":{\"serial_event_id\":{\"from\":3.1,\"to\":4.5,\"include_lower\":true,\"include_upper\":false"
-                    }
-            },
-            {"notFilter", "process where not (exit_code > -1)",
-                    "\"range\":{\"exit_code\":{\"from\":null,\"to\":-1,\"include_lower\":false,\"include_upper\":true"
-            },
-            {"inFilter", "process where process_name in (\"python.exe\", \"SMSS.exe\", \"explorer.exe\")",
-                    new Object[]{
-                            "\"term\":{\"process_name\":{\"value\":\"python.exe\"",
-                            "\"term\":{\"process_name\":{\"value\":\"SMSS.exe\"",
-                            "\"term\":{\"process_name\":{\"value\":\"explorer.exe\"",
-                    }
-            },
-            {"equalsAndInFilter", "process where process_path == \"*\\\\red_ttp\\\\wininit.*\" and opcode in (0,1,2,3)",
-                    new Object[]{
-                            "\"wildcard\":{\"process_path\":{\"wildcard\":\"*\\\\\\\\red_ttp\\\\\\\\wininit.*\"",
-                            "\"term\":{\"opcode\":{\"value\":0",
-                            "\"term\":{\"opcode\":{\"value\":1",
-                            "\"term\":{\"opcode\":{\"value\":2",
-                            "\"term\":{\"opcode\":{\"value\":3",
-                    }
-            },
-    };
-
-    private final String name;
-    private final String query;
-    private final Object expect;
-
-    public QueryFolderOkTests(String name, String query, Object expect) {
-        this.name = name;
-        this.query = query;
-        this.expect = expect;
-    }
-
-    @ParametersFactory(shuffle = false, argumentFormatting = "%1$s.test")
-    public static Iterable<Object[]> parameters() {
-        return Arrays.asList(specs);
-    }
-
-    public void test() {
-        PhysicalPlan p = plan(query);
-        assertEquals(EsQueryExec.class, p.getClass());
-        EsQueryExec eqe = (EsQueryExec) p;
-        assertEquals(23, eqe.output().size());
-        assertEquals(KEYWORD, eqe.output().get(0).dataType());
-
-        final String query = eqe.queryContainer().toString().replaceAll("\\s+", "");
-
-        // test query term
-        if (expect != null) {
-            if (expect instanceof Object[]) {
-                for (Object item : (Object[]) expect) {
-                    assertThat(query, containsString((String) item));
-                }
-            } else {
-                assertThat(query, containsString((String) expect));
-            }
-        }
-
-        // test common term
-        assertThat(query, containsString("\"term\":{\"event.category\":{\"value\":\"process\""));
-
-        // test field source extraction
-        assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]"));
-    }
-}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java
new file mode 100644
index 0000000000000..1c0664ee7ac99
--- /dev/null
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.planner;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.eql.analysis.Analyzer;
+import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
+import org.elasticsearch.xpack.eql.analysis.Verifier;
+import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
+import org.elasticsearch.xpack.eql.optimizer.Optimizer;
+import org.elasticsearch.xpack.eql.parser.EqlParser;
+import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
+import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
+import org.elasticsearch.xpack.ql.index.EsIndex;
+import org.elasticsearch.xpack.ql.index.IndexResolution;
+
+import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.ql.type.TypesTests.loadMapping;
+import static org.hamcrest.Matchers.containsString;
+
+public class QueryFolderTests extends ESTestCase {
+
+    private EqlParser parser = new EqlParser();
+    private PreAnalyzer preAnalyzer = new PreAnalyzer();
+    private Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier());
+    private Optimizer optimizer = new Optimizer();
+    private Planner planner = new Planner();
+
+    private IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json")));
+
+
+    private PhysicalPlan plan(IndexResolution resolution, String eql) {
+        return planner.plan(optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))));
+    }
+
+    private PhysicalPlan plan(String eql) {
+        return plan(index, eql);
+    }
+    
+    public void testBasicPlan() {
+        PhysicalPlan p = plan("process where true");
+        assertEquals(EsQueryExec.class, p.getClass());
+        EsQueryExec eqe = (EsQueryExec) p;
+        assertEquals(23, eqe.output().size());
+        assertEquals(KEYWORD, eqe.output().get(0).dataType());
+        String query = eqe.queryContainer().toString().replaceAll("\\s+", "");
+        // test query term
+        assertThat(query, containsString("\"term\":{\"event.category\":{\"value\":\"process\""));
+        // test field source extraction
+        assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]"));
+    }
+}
diff --git a/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml
index b510f80d37f9f..f8a96eaef12aa 100644
--- a/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml
+++ b/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml
@@ -12,11 +12,14 @@
 # query = 'process where serial_event_id = 1'
 # expected_event_ids  = [1]
 
+[[queries]]
+query = 'process where serial_event_id < 4'
+expected_event_ids  = [1, 2, 3]
+
 [[queries]]
 query = 'process where true | head 6'
 expected_event_ids  = [1, 2, 3, 4, 5, 6]
 
-# presently not supported, throwing: org.elasticsearch.xpack.ql.rule.RuleExecutionException: Does not know how to handle a local relation
 [[queries]]
 query = 'process where false'
 expected_event_ids = []
@@ -29,6 +32,12 @@ query = 'process where missing_field != null'
 expected_event_ids  = [1, 2, 3, 4, 5]
 query = 'process where bad_field == null | head 5'
 
+[[queries]]
+query = '''
+  process where process_name == "impossible name" or (serial_event_id < 4.5 and serial_event_id >= 3.1)
+'''
+expected_event_ids  = [4]
+
 [[queries]]
 tags = ["comparisons", "pipes"]
 query = '''
@@ -61,6 +70,42 @@ process where true
 '''
 expected_event_ids  = [9, 10]
 
+[[queries]]
+query = '''
+process where serial_event_id<=8 and serial_event_id > 7
+'''
+expected_event_ids  = [8]
+
+[[queries]]
+note = "check that comparisons against null values return false"
+expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
+query = 'process where exit_code >= 0'
+
+[[queries]]
+note = "check that comparisons against null values return false"
+expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
+query = 'process where 0 <= exit_code'
+
+[[queries]]
+note = "check that comparisons against null values return false"
+expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
+query = 'process where exit_code <= 0'
+
+[[queries]]
+note = "check that comparisons against null values return false"
+expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
+query = 'process where exit_code < 1'
+
+[[queries]]
+note = "check that comparisons against null values return false"
+expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
+query = 'process where exit_code > -1'
+
+[[queries]]
+note = "check that comparisons against null values return false"
+expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
+query = 'process where -1 < exit_code'
+
 [[queries]]
 note = "check that comparisons against null values return false"
 expected_event_ids = []
@@ -80,6 +125,26 @@ note = "check that comparisons against null values return false"
 expected_event_ids  = [1, 2, 3, 4, 5, 6, 7]
 query = 'process where not (-1 < exit_code) | head 7'
 
+[[queries]]
+query = 'process where exit_code > 0'
+expected_event_ids = []
+
+[[queries]]
+query = 'process where exit_code < 0'
+expected_event_ids = []
+
+[[queries]]
+query = 'process where 0 < exit_code'
+expected_event_ids = []
+
+[[queries]]
+query = 'process where 0 > exit_code'
+expected_event_ids = []
+
+[[queries]]
+query = 'process where (serial_event_id<=8 and serial_event_id > 7) and (opcode=3 and opcode>2)'
+expected_event_ids  = [8]
+
 [[queries]]
 query = 'process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)'
 expected_event_ids  = [7, 8]
@@ -152,6 +217,16 @@ expected_event_ids  = [57]
 query = '''
 registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] == "EN"'''
 
+[[queries]]
+query = '''
+registry where key_path == "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"'''
+expected_event_ids  = [79]
+
+[[queries]]
+query = '''
+process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)'''
+expected_event_ids  = [84, 85]
+
 [[queries]]
 query = '''
 file where file_name == "csrss.exe" and opcode=0
@@ -178,6 +253,34 @@ process where opcode=1 and process_name == "smss.exe"
 '''
 expected_event_ids  = [78]
 
+[[queries]]
+query = '''
+file where file_path="*\\red_ttp\\winin*.*"
+  and opcode in (0,1,2) and user_name="vagrant"
+'''
+expected_event_ids  = [83, 86]
+
+[[queries]]
+query = '''
+file where file_path="*\\red_ttp\\winin*.*"
+  and opcode not in (0,1,2) and user_name="vagrant"
+'''
+expected_event_ids  = []
+
+[[queries]]
+query = '''
+file where file_path="*\\red_ttp\\winin*.*"
+  and opcode not in (3, 4, 5, 6 ,7) and user_name="vagrant"
+'''
+expected_event_ids  = [83, 86]
+
+
+[[queries]]
+query = '''
+file where file_name in ("wininit.exe", "lsass.exe") and opcode == 2
+'''
+expected_event_ids  = [65, 86]
+
 [[queries]]
 query = '''
 file where true
@@ -509,6 +612,15 @@ query = '''
 registry where length(bad_field) > 0
 '''
 
+[[queries]]
+query = '''
+process where opcode == 1
+  and process_name in ("net.exe", "net1.exe")
+  and not (parent_process_name == "net.exe"
+  and process_name == "net1.exe")
+  and command_line == "*group *admin*" and command_line != "* /add*"'''
+expected_event_ids  = [97]
+
 [[queries]]
 expected_event_ids  = [1, 55, 57, 63, 75304]
 query = '''
@@ -578,6 +690,11 @@ file where event of [process where process_name = "python.exe" ]
 | unique unique_pid'''
 expected_event_ids  = [55, 95]
 
+[[queries]]
+query = '''
+process where process_name = "python.exe"'''
+expected_event_ids  = [48, 50, 51, 54, 93]
+
 [[queries]]
 query = 'process where event of [process where process_name = "python.exe" ]'
 expected_event_ids  = [48, 50, 51, 54, 93]
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index d043117412e56..5a22857494557 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
@@ -70,8 +69,7 @@
 public final class FrozenEngine extends ReadOnlyEngine {
     public static final Setting<Boolean> INDEX_FROZEN = Setting.boolSetting("index.frozen", false, Setting.Property.IndexScope,
         Setting.Property.PrivateIndex);
-    private final SegmentsStats segmentsStats;
-    private final DocsStats docsStats;
+    private final SegmentsStats stats;
     private volatile ElasticsearchDirectoryReader lastOpenedReader;
     private final ElasticsearchDirectoryReader canMatchReader;
 
@@ -80,15 +78,15 @@ public FrozenEngine(EngineConfig config) {
 
         boolean success = false;
         Directory directory = store.directory();
-        try (DirectoryReader reader = openDirectory(directory)) {
-            // we record the segment stats and doc stats here - that's what the reader needs when it's open and it give the user
+        // Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
+        try (DirectoryReader reader = openDirectory(directory, false)) {
+            // we record the segment stats here - that's what the reader needs when it's open and it give the user
             // an idea of what it can save when it's closed
-            this.segmentsStats = new SegmentsStats();
+            this.stats = new SegmentsStats();
             for (LeafReaderContext ctx : reader.getContext().leaves()) {
                 SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
-                fillSegmentStats(segmentReader, true, segmentsStats);
+                fillSegmentStats(segmentReader, true, stats);
             }
-            this.docsStats = docsStats(reader);
             final DirectoryReader wrappedReader = new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
             canMatchReader = ElasticsearchDirectoryReader.wrap(
                 new RewriteCachingDirectoryReader(directory, wrappedReader.leaves()), config.getShardId());
@@ -172,7 +170,7 @@ private synchronized ElasticsearchDirectoryReader getOrOpenReader() throws IOExc
                 for (ReferenceManager.RefreshListener listeners : config ().getInternalRefreshListener()) {
                     listeners.beforeRefresh();
                 }
-                final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory());
+                final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory(), true);
                 reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
                 processReader(reader);
                 reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);
@@ -596,7 +594,7 @@ public LeafReader getDelegate() {
     public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
         if (includeUnloadedSegments) {
             final SegmentsStats stats = new SegmentsStats();
-            stats.add(this.segmentsStats);
+            stats.add(this.stats);
             if (includeSegmentFileSizes == false) {
                 stats.clearFileSizes();
             }
@@ -607,11 +605,6 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
 
     }
 
-    @Override
-    public DocsStats docStats() {
-        return docsStats;
-    }
-
     synchronized boolean isReaderOpen() {
         return lastOpenedReader != null;
     } // this is mainly for tests
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 3c53b78e4523b..00414b20679e3 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.xpack.core.ilm.AllocateAction;
 import org.elasticsearch.xpack.core.ilm.DeleteAction;
 import org.elasticsearch.xpack.core.ilm.DeleteStep;
+import org.elasticsearch.xpack.core.ilm.ErrorStep;
 import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
 import org.elasticsearch.xpack.core.ilm.FreezeAction;
 import org.elasticsearch.xpack.core.ilm.FreezeStep;
@@ -1395,7 +1396,6 @@ public void testHistoryIsWrittenWithSuccess() throws Exception {
         assertBusy(() -> assertHistoryIsPresent(policy, index + "-000002", true, "check-rollover-ready"), 30, TimeUnit.SECONDS);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50353")
     public void testHistoryIsWrittenWithFailure() throws Exception {
         createIndexWithSettings(index + "-1", Settings.builder(), false);
         createNewSingletonPolicy("hot", new RolloverAction(null, null, 1L));
@@ -1406,8 +1406,7 @@ public void testHistoryIsWrittenWithFailure() throws Exception {
         Request refreshIndex = new Request("POST", "/" + index + "-1/_refresh");
         client().performRequest(refreshIndex);
 
-        // Check that we've had error and auto retried
-        assertBusy(() -> assertThat((Integer) explainIndex(index + "-1").get("failed_step_retry_count"), greaterThanOrEqualTo(1)));
+        assertBusy(() -> assertThat(getStepKeyForIndex(index + "-1").getName(), equalTo(ErrorStep.NAME)), 30, TimeUnit.SECONDS);
 
         assertBusy(() -> assertHistoryIsPresent(policy, index + "-1", false, "ERROR"), 30, TimeUnit.SECONDS);
     }
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
index f2c9e9c609502..de77dbed9f7f4 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
@@ -91,7 +91,6 @@ public void testMissingRepo() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50358")
     public void testFullPolicySnapshot() throws Exception {
         final String indexName = "test";
         final String policyName = "full-policy";
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index 965c5beb5af04..4f4c2913fb2a4 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -100,6 +100,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
+        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
 
         // This is necessary to prevent ILM and SLM installing a lifecycle policy, these tests assume a blank slate
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
index 58f0b632e1321..180ff53cf5b97 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
@@ -50,6 +50,7 @@ protected Settings nodeSettings() {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
+        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         settings.put(Environment.PATH_REPO_SETTING.getKey(), repositoryLocation);
 
         settings.put(XPackSettings.SNAPSHOT_LIFECYCLE_ENABLED.getKey(), true);
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
index b3149cc177e5a..6523d7a07228a 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.slm;
 
-import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -31,18 +30,6 @@ public class SnapshotLifecyclePolicyTests extends AbstractSerializingTestCase<Sn
 
     private String id;
 
-    public void testToRequest() {
-        SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy("id", "name", "0 1 2 3 4 ? 2099", "repo", Collections.emptyMap(),
-            SnapshotRetentionConfiguration.EMPTY);
-        CreateSnapshotRequest request = p.toRequest();
-        CreateSnapshotRequest expected = new CreateSnapshotRequest().userMetadata(Collections.singletonMap("policy", "id"));
-
-        p = new SnapshotLifecyclePolicy("id", "name", "0 1 2 3 4 ? 2099", "repo", null, null);
-        request = p.toRequest();
-        expected.waitForCompletion(true).snapshot(request.snapshot()).repository("repo");
-        assertEquals(expected, request);
-    }
-
     public void testNameGeneration() {
         long time = 1552684146542L; // Fri Mar 15 2019 21:09:06 UTC
         SnapshotLifecyclePolicy.ResolverContext context = new SnapshotLifecyclePolicy.ResolverContext(time);
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
index 3d22402e89779..177135974fb6c 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
@@ -36,27 +37,24 @@ public class Logstash extends Plugin implements SystemIndexPlugin {
     private static final String OLD_LOGSTASH_INDEX_NAME = "logstash-index-template";
     private static final String TEMPLATE_VERSION_VARIABLE = "logstash.template.version";
 
-    public Logstash() {}
+    private final boolean enabled;
+
+    public Logstash(Settings settings) {
+        this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings);
+    }
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
             new ActionHandler<>(XPackUsageFeatureAction.LOGSTASH, LogstashUsageTransportAction.class),
-            new ActionHandler<>(XPackInfoFeatureAction.LOGSTASH, LogstashInfoTransportAction.class)
-        );
+            new ActionHandler<>(XPackInfoFeatureAction.LOGSTASH, LogstashInfoTransportAction.class));
     }
 
     public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
         return templates -> {
             templates.keySet().removeIf(OLD_LOGSTASH_INDEX_NAME::equals);
-            TemplateUtils.loadTemplateIntoMap(
-                "/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json",
-                templates,
-                LOGSTASH_INDEX_TEMPLATE_NAME,
-                Version.CURRENT.toString(),
-                TEMPLATE_VERSION_VARIABLE,
-                LogManager.getLogger(Logstash.class)
-            );
+            TemplateUtils.loadTemplateIntoMap("/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json", templates, LOGSTASH_INDEX_TEMPLATE_NAME,
+                    Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE, LogManager.getLogger(Logstash.class));
             assert templates.get(LOGSTASH_INDEX_TEMPLATE_NAME).mappings() != null;
             return templates;
         };
@@ -64,8 +62,7 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return Collections.singletonList(
-            new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME, "Contains data for Logstash Central Management")
-        );
+        return Collections.singletonList(new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME,
+            "Contains data for Logstash Central Management"));
     }
 }
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
index 4aa77af2988f4..49a48daf5ea9c 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
@@ -21,12 +21,8 @@ public class LogstashInfoTransportAction extends XPackInfoFeatureTransportAction
     private final XPackLicenseState licenseState;
 
     @Inject
-    public LogstashInfoTransportAction(
-        TransportService transportService,
-        ActionFilters actionFilters,
-        Settings settings,
-        XPackLicenseState licenseState
-    ) {
+    public LogstashInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                       Settings settings, XPackLicenseState licenseState) {
         super(XPackInfoFeatureAction.LOGSTASH.name(), transportService, actionFilters);
         this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings);
         this.licenseState = licenseState;
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
index e03cc822ec746..55f096ec2e8e9 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
@@ -29,36 +29,21 @@ public class LogstashUsageTransportAction extends XPackUsageFeatureTransportActi
     private final XPackLicenseState licenseState;
 
     @Inject
-    public LogstashUsageTransportAction(
-        TransportService transportService,
-        ClusterService clusterService,
-        ThreadPool threadPool,
-        ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        Settings settings,
-        XPackLicenseState licenseState
-    ) {
-        super(
-            XPackUsageFeatureAction.LOGSTASH.name(),
-            transportService,
-            clusterService,
-            threadPool,
-            actionFilters,
-            indexNameExpressionResolver
-        );
+    public LogstashUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                        Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.LOGSTASH.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
         this.settings = settings;
         this.licenseState = licenseState;
     }
 
     @Override
-    protected void masterOperation(
-        Task task,
-        XPackUsageRequest request,
-        ClusterState state,
-        ActionListener<XPackUsageFeatureResponse> listener
-    ) {
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
         boolean available = licenseState.isLogstashAllowed();
-        LogstashFeatureSetUsage usage = new LogstashFeatureSetUsage(available, XPackSettings.LOGSTASH_ENABLED.get(settings));
+        LogstashFeatureSetUsage usage =
+            new LogstashFeatureSetUsage(available, XPackSettings.LOGSTASH_ENABLED.get(settings));
         listener.onResponse(new XPackUsageFeatureResponse(usage));
     }
 }
diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
index e3872352b944d..42115fbf74c07 100644
--- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
+++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
@@ -8,13 +8,11 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
 
@@ -26,13 +24,12 @@ public class LogstashInfoTransportActionTests extends ESTestCase {
 
     public void testEnabledSetting() throws Exception {
         boolean enabled = randomBoolean();
-        Settings settings = Settings.builder().put("path.home", createTempDir()).put("xpack.logstash.enabled", enabled).build();
+        Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.logstash.enabled", enabled)
+            .build();
         LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
-            mock(TransportService.class),
-            mock(ActionFilters.class),
-            settings,
-            null
-        );
+            mock(TransportService.class), mock(ActionFilters.class), settings, null);
         assertThat(featureSet.enabled(), is(enabled));
 
         LogstashUsageTransportAction usageAction = newUsageAction(settings, false);
@@ -44,29 +41,19 @@ public void testEnabledSetting() throws Exception {
         usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new LogstashFeatureSetUsage(out.bytes().streamInput());
         assertThat(serializedUsage.enabled(), is(enabled));
-
-        assertSettingDeprecationsAndWarnings(new Setting<?>[] { XPackSettings.LOGSTASH_ENABLED });
     }
 
     public void testEnabledDefault() throws Exception {
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
         LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
-            mock(TransportService.class),
-            mock(ActionFilters.class),
-            settings,
-            null
-        );
+            mock(TransportService.class), mock(ActionFilters.class), settings, null);
         assertThat(featureSet.enabled(), is(true));
     }
 
     public void testAvailable() throws Exception {
         final XPackLicenseState licenseState = mock(XPackLicenseState.class);
         LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
-            mock(TransportService.class),
-            mock(ActionFilters.class),
-            Settings.EMPTY,
-            licenseState
-        );
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isLogstashAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -86,14 +73,7 @@ public void testAvailable() throws Exception {
     private LogstashUsageTransportAction newUsageAction(Settings settings, boolean available) {
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isLogstashAllowed()).thenReturn(available);
-        return new LogstashUsageTransportAction(
-            mock(TransportService.class),
-            null,
-            null,
-            mock(ActionFilters.class),
-            null,
-            settings,
-            licenseState
-        );
+        return new LogstashUsageTransportAction(mock(TransportService.class), null,
+            null, mock(ActionFilters.class), null, settings, licenseState);
     }
 }
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index e4702bb0245c0..d0383b0999e8d 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -50,7 +50,6 @@ dependencies {
   compileOnly project(':modules:lang-painless:spi')
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-  testCompile project(path: xpackModule('ilm'), configuration: 'default')
   // This should not be here
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
 
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index 516e5c626fca2..2ac378e472a7b 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -76,8 +76,8 @@ integTest.runner {
     'ml/data_frame_analytics_crud/Test put regression given negative gamma',
     'ml/data_frame_analytics_crud/Test put regression given eta less than 1e-3',
     'ml/data_frame_analytics_crud/Test put regression given eta greater than one',
-    'ml/data_frame_analytics_crud/Test put regression given max_trees is zero',
-    'ml/data_frame_analytics_crud/Test put regression given max_trees is greater than 2k',
+    'ml/data_frame_analytics_crud/Test put regression given maximum_number_trees is zero',
+    'ml/data_frame_analytics_crud/Test put regression given maximum_number_trees is greater than 2k',
     'ml/data_frame_analytics_crud/Test put regression given feature_bag_fraction is negative',
     'ml/data_frame_analytics_crud/Test put regression given feature_bag_fraction is greater than one',
     'ml/data_frame_analytics_crud/Test put regression given training_percent is less than one',
@@ -87,8 +87,8 @@ integTest.runner {
     'ml/data_frame_analytics_crud/Test put classification given negative gamma',
     'ml/data_frame_analytics_crud/Test put classification given eta less than 1e-3',
     'ml/data_frame_analytics_crud/Test put classification given eta greater than one',
-    'ml/data_frame_analytics_crud/Test put classification given max_trees is zero',
-    'ml/data_frame_analytics_crud/Test put classification given max_trees is greater than 2k',
+    'ml/data_frame_analytics_crud/Test put classification given maximum_number_trees is zero',
+    'ml/data_frame_analytics_crud/Test put classification given maximum_number_trees is greater than 2k',
     'ml/data_frame_analytics_crud/Test put classification given feature_bag_fraction is negative',
     'ml/data_frame_analytics_crud/Test put classification given feature_bag_fraction is greater than one',
     'ml/data_frame_analytics_crud/Test put classification given num_top_classes is less than zero',
@@ -139,9 +139,6 @@ integTest.runner {
     'ml/inference_crud/Test put ensemble with empty models',
     'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index',
     'ml/inference_crud/Test put model with empty input.field_names',
-    'ml/inference_processor/Test create processor with missing mandatory fields',
-    'ml/inference_processor/Test create and delete pipeline with inference processor',
-    'ml/inference_processor/Test create processor with deprecated fields',
     'ml/inference_stats_crud/Test get stats given missing trained model',
     'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false',
     'ml/jobs_crud/Test cannot create job with existing categorizer state document',
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
index 04f71e0c6f7d4..4ca8fb5a64373 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -79,7 +80,7 @@ public void setUpData() {
     public void tearDownData() {
         cleanUp();
         client().admin().indices().prepareDelete(DATA_INDEX).get();
-        refresh("*");
+        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
     }
 
     public void testBasicCategorization() throws Exception {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
index 773f91b5027d2..ccf39aa8aa472 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
@@ -5,21 +5,18 @@
  */
 package org.elasticsearch.xpack.ml.integration;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.aggregations.MultiBucketConsumerService.TooManyBucketsException;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Classification;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Classification;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix;
 import org.junit.After;
 import org.junit.Before;
 
@@ -30,20 +27,16 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
 
 public class ClassificationEvaluationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
 
     private static final String ANIMALS_DATA_INDEX = "test-evaluate-animals-index";
 
-    private static final String ANIMAL_NAME_KEYWORD_FIELD = "animal_name_keyword";
+    private static final String ANIMAL_NAME_FIELD = "animal_name";
     private static final String ANIMAL_NAME_PREDICTION_FIELD = "animal_name_prediction";
-    private static final String NO_LEGS_KEYWORD_FIELD = "no_legs_keyword";
-    private static final String NO_LEGS_INTEGER_FIELD = "no_legs_integer";
+    private static final String NO_LEGS_FIELD = "no_legs";
     private static final String NO_LEGS_PREDICTION_FIELD = "no_legs_prediction";
-    private static final String IS_PREDATOR_KEYWORD_FIELD = "predator_keyword";
-    private static final String IS_PREDATOR_BOOLEAN_FIELD = "predator_boolean";
+    private static final String IS_PREDATOR_FIELD = "predator";
     private static final String IS_PREDATOR_PREDICTION_FIELD = "predator_prediction";
 
     @Before
@@ -55,15 +48,11 @@ public void setup() {
     @After
     public void cleanup() {
         cleanUp();
-        client().admin().cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder().putNull("search.max_buckets"))
-            .get();
     }
 
     public void testEvaluate_DefaultMetrics() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, null));
+            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, null));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(
@@ -76,7 +65,7 @@ public void testEvaluate_AllMetrics() {
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
                 new Classification(
-                    ANIMAL_NAME_KEYWORD_FIELD,
+                    ANIMAL_NAME_FIELD,
                     ANIMAL_NAME_PREDICTION_FIELD,
                     List.of(new Accuracy(), new MulticlassConfusionMatrix(), new Precision(), new Recall())));
 
@@ -93,7 +82,7 @@ public void testEvaluate_AllMetrics() {
     public void testEvaluate_Accuracy_KeywordField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Accuracy())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -112,9 +101,10 @@ public void testEvaluate_Accuracy_KeywordField() {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(5.0 / 75));
     }
 
-    private void evaluateAccuracy_IntegerField(String actualField) {
+    public void testEvaluate_Accuracy_IntegerField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_PREDICTION_FIELD, List.of(new Accuracy())));
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(NO_LEGS_FIELD, NO_LEGS_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -133,18 +123,10 @@ private void evaluateAccuracy_IntegerField(String actualField) {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(15.0 / 75));
     }
 
-    public void testEvaluate_Accuracy_IntegerField() {
-        evaluateAccuracy_IntegerField(NO_LEGS_INTEGER_FIELD);
-    }
-
-    public void testEvaluate_Accuracy_IntegerField_MappingTypeMismatch() {
-        evaluateAccuracy_IntegerField(NO_LEGS_KEYWORD_FIELD);
-    }
-
-    private void evaluateAccuracy_BooleanField(String actualField) {
+    public void testEvaluate_Accuracy_BooleanField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Accuracy())));
+                ANIMALS_DATA_INDEX, new Classification(IS_PREDATOR_FIELD, IS_PREDATOR_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -160,18 +142,10 @@ private void evaluateAccuracy_BooleanField(String actualField) {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(45.0 / 75));
     }
 
-    public void testEvaluate_Accuracy_BooleanField() {
-        evaluateAccuracy_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
-    }
-
-    public void testEvaluate_Accuracy_BooleanField_MappingTypeMismatch() {
-        evaluateAccuracy_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
-    }
-
-    public void testEvaluate_Precision_KeywordField() {
+    public void testEvaluate_Precision() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -190,78 +164,20 @@ public void testEvaluate_Precision_KeywordField() {
         assertThat(precisionResult.getAvgPrecision(), equalTo(5.0 / 75));
     }
 
-    private void evaluatePrecision_IntegerField(String actualField) {
-        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_PREDICTION_FIELD, List.of(new Precision())));
-
-        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
-        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
-
-        Precision.Result precisionResult = (Precision.Result) evaluateDataFrameResponse.getMetrics().get(0);
-        assertThat(precisionResult.getMetricName(), equalTo(Precision.NAME.getPreferredName()));
-        assertThat(
-            precisionResult.getClasses(),
-            equalTo(
-                List.of(
-                    new Precision.PerClassResult("1", 0.2),
-                    new Precision.PerClassResult("2", 0.2),
-                    new Precision.PerClassResult("3", 0.2),
-                    new Precision.PerClassResult("4", 0.2),
-                    new Precision.PerClassResult("5", 0.2))));
-        assertThat(precisionResult.getAvgPrecision(), equalTo(0.2));
-    }
-
-    public void testEvaluate_Precision_IntegerField() {
-        evaluatePrecision_IntegerField(NO_LEGS_INTEGER_FIELD);
-    }
-
-    public void testEvaluate_Precision_IntegerField_MappingTypeMismatch() {
-        evaluatePrecision_IntegerField(NO_LEGS_KEYWORD_FIELD);
-    }
-
-    private void evaluatePrecision_BooleanField(String actualField) {
-        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Precision())));
-
-        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
-        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
-
-        Precision.Result precisionResult = (Precision.Result) evaluateDataFrameResponse.getMetrics().get(0);
-        assertThat(precisionResult.getMetricName(), equalTo(Precision.NAME.getPreferredName()));
-        assertThat(
-            precisionResult.getClasses(),
-            equalTo(
-                List.of(
-                    new Precision.PerClassResult("false", 0.5),
-                    new Precision.PerClassResult("true", 9.0 / 13))));
-        assertThat(precisionResult.getAvgPrecision(), equalTo(31.0 / 52));
-    }
-
-    public void testEvaluate_Precision_BooleanField() {
-        evaluatePrecision_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
-    }
-
-    public void testEvaluate_Precision_BooleanField_MappingTypeMismatch() {
-        evaluatePrecision_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
-    }
-
     public void testEvaluate_Precision_CardinalityTooHigh() {
         indexDistinctAnimals(ANIMALS_DATA_INDEX, 1001);
         ElasticsearchStatusException e =
             expectThrows(
                 ElasticsearchStatusException.class,
                 () -> evaluateDataFrame(
-                    ANIMALS_DATA_INDEX,
-                    new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision()))));
-        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name_keyword] is too high"));
+                    ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision()))));
+        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name] is too high"));
     }
 
-    public void testEvaluate_Recall_KeywordField() {
+    public void testEvaluate_Recall() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -280,79 +196,21 @@ public void testEvaluate_Recall_KeywordField() {
         assertThat(recallResult.getAvgRecall(), equalTo(5.0 / 75));
     }
 
-    private void evaluateRecall_IntegerField(String actualField) {
-        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_INTEGER_FIELD, List.of(new Recall())));
-
-        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
-        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
-
-        Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(0);
-        assertThat(recallResult.getMetricName(), equalTo(Recall.NAME.getPreferredName()));
-        assertThat(
-            recallResult.getClasses(),
-            equalTo(
-                List.of(
-                    new Recall.PerClassResult("1", 1.0),
-                    new Recall.PerClassResult("2", 1.0),
-                    new Recall.PerClassResult("3", 1.0),
-                    new Recall.PerClassResult("4", 1.0),
-                    new Recall.PerClassResult("5", 1.0))));
-        assertThat(recallResult.getAvgRecall(), equalTo(1.0));
-    }
-
-    public void testEvaluate_Recall_IntegerField() {
-        evaluateRecall_IntegerField(NO_LEGS_INTEGER_FIELD);
-    }
-
-    public void testEvaluate_Recall_IntegerField_MappingTypeMismatch() {
-        evaluateRecall_IntegerField(NO_LEGS_KEYWORD_FIELD);
-    }
-
-    private void evaluateRecall_BooleanField(String actualField) {
-        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Recall())));
-
-        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
-        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
-
-        Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(0);
-        assertThat(recallResult.getMetricName(), equalTo(Recall.NAME.getPreferredName()));
-        assertThat(
-            recallResult.getClasses(),
-            equalTo(
-                List.of(
-                    new Recall.PerClassResult("true", 0.6),
-                    new Recall.PerClassResult("false", 0.6))));
-        assertThat(recallResult.getAvgRecall(), equalTo(0.6));
-    }
-
-    public void testEvaluate_Recall_BooleanField() {
-        evaluateRecall_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
-    }
-
-    public void testEvaluate_Recall_BooleanField_MappingTypeMismatch() {
-        evaluateRecall_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
-    }
-
     public void testEvaluate_Recall_CardinalityTooHigh() {
         indexDistinctAnimals(ANIMALS_DATA_INDEX, 1001);
         ElasticsearchStatusException e =
             expectThrows(
                 ElasticsearchStatusException.class,
                 () -> evaluateDataFrame(
-                    ANIMALS_DATA_INDEX,
-                    new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall()))));
-        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name_keyword] is too high"));
+                    ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall()))));
+        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name] is too high"));
     }
 
-    private void evaluateWithMulticlassConfusionMatrix() {
+    public void testEvaluate_ConfusionMatrixMetricWithDefaultSize() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
-                new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix())));
+                new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -411,29 +269,11 @@ private void evaluateWithMulticlassConfusionMatrix() {
         assertThat(confusionMatrixResult.getOtherActualClassCount(), equalTo(0L));
     }
 
-    public void testEvaluate_ConfusionMatrixMetricWithDefaultSize() {
-        evaluateWithMulticlassConfusionMatrix();
-
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("search.max_buckets", 20)).get();
-        evaluateWithMulticlassConfusionMatrix();
-
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("search.max_buckets", 7)).get();
-        evaluateWithMulticlassConfusionMatrix();
-
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("search.max_buckets", 6)).get();
-        ElasticsearchException e = expectThrows(ElasticsearchException.class, this::evaluateWithMulticlassConfusionMatrix);
-
-        assertThat(e.getCause(), is(instanceOf(TooManyBucketsException.class)));
-        TooManyBucketsException tmbe = (TooManyBucketsException) e.getCause();
-        assertThat(tmbe.getMaxBuckets(), equalTo(6));
-    }
-
     public void testEvaluate_ConfusionMatrixMetricWithUserProvidedSize() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
-                new Classification(
-                    ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix(3, null))));
+                new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix(3, null))));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -470,13 +310,11 @@ public void testEvaluate_ConfusionMatrixMetricWithUserProvidedSize() {
     private static void createAnimalsIndex(String indexName) {
         client().admin().indices().prepareCreate(indexName)
             .setMapping(
-                ANIMAL_NAME_KEYWORD_FIELD, "type=keyword",
+                ANIMAL_NAME_FIELD, "type=keyword",
                 ANIMAL_NAME_PREDICTION_FIELD, "type=keyword",
-                NO_LEGS_KEYWORD_FIELD, "type=keyword",
-                NO_LEGS_INTEGER_FIELD, "type=integer",
+                NO_LEGS_FIELD, "type=integer",
                 NO_LEGS_PREDICTION_FIELD, "type=integer",
-                IS_PREDATOR_KEYWORD_FIELD, "type=keyword",
-                IS_PREDATOR_BOOLEAN_FIELD, "type=boolean",
+                IS_PREDATOR_FIELD, "type=boolean",
                 IS_PREDATOR_PREDICTION_FIELD, "type=boolean")
             .get();
     }
@@ -491,13 +329,11 @@ private static void indexAnimalsData(String indexName) {
                     bulkRequestBuilder.add(
                         new IndexRequest(indexName)
                             .source(
-                                ANIMAL_NAME_KEYWORD_FIELD, animalNames.get(i),
+                                ANIMAL_NAME_FIELD, animalNames.get(i),
                                 ANIMAL_NAME_PREDICTION_FIELD, animalNames.get((i + j) % animalNames.size()),
-                                NO_LEGS_KEYWORD_FIELD, String.valueOf(i + 1),
-                                NO_LEGS_INTEGER_FIELD, i + 1,
+                                NO_LEGS_FIELD, i + 1,
                                 NO_LEGS_PREDICTION_FIELD, j + 1,
-                                IS_PREDATOR_KEYWORD_FIELD, String.valueOf(i % 2 == 0),
-                                IS_PREDATOR_BOOLEAN_FIELD, i % 2 == 0,
+                                IS_PREDATOR_FIELD, i % 2 == 0,
                                 IS_PREDATOR_PREDICTION_FIELD, (i + j) % 2 == 0));
                 }
             }
@@ -513,8 +349,7 @@ private static void indexDistinctAnimals(String indexName, int distinctAnimalCou
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         for (int i = 0; i < distinctAnimalCount; i++) {
             bulkRequestBuilder.add(
-                new IndexRequest(indexName)
-                    .source(ANIMAL_NAME_KEYWORD_FIELD, "animal_" + i, ANIMAL_NAME_PREDICTION_FIELD, randomAlphaOfLength(5)));
+                new IndexRequest(indexName).source(ANIMAL_NAME_FIELD, "animal_" + i, ANIMAL_NAME_PREDICTION_FIELD, randomAlphaOfLength(5)));
         }
         BulkResponse bulkResponse = bulkRequestBuilder.get();
         if (bulkResponse.hasFailures()) {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index a58418bbd35ef..8c029c5f84bb2 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -31,6 +32,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall;
 import org.junit.After;
+import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -70,9 +72,25 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
     private String destIndex;
     private boolean analysisUsesExistingDestIndex;
 
+    @Before
+    public void enableLogging() {
+        client().admin().cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder()
+                .put("logger.org.elasticsearch.xpack.ml.action", "DEBUG")
+                .put("logger.org.elasticsearch.xpack.ml.dataframe", "DEBUG"))
+            .get();
+    }
+
     @After
     public void cleanup() {
         cleanUp();
+        client().admin().cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder()
+                .putNull("logger.org.elasticsearch.xpack.ml.action")
+                .putNull("logger.org.elasticsearch.xpack.ml.dataframe"))
+            .get();
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
@@ -88,7 +106,6 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
                 null,
                 null,
                 null,
-                null,
                 null));
         registerAnalytics(config);
         putAnalytics(config);
@@ -190,7 +207,7 @@ public <T> void testWithOnlyTrainingRowsAndTrainingPercentIsFifty(String jobId,
                 sourceIndex,
                 destIndex,
                 null,
-                new Classification(dependentVariable, BoostedTreeParams.builder().build(), null, null, numTopClasses, 50.0, null));
+                new Classification(dependentVariable, BoostedTreeParams.builder().build(), null, numTopClasses, 50.0, null));
         registerAnalytics(config);
         putAnalytics(config);
 
@@ -322,11 +339,9 @@ public void testStopAndRestart() throws Exception {
         assertEvaluation(KEYWORD_FIELD, KEYWORD_FIELD_VALUES, "ml." + predictedClassField);
     }
 
-    @AwaitsFix(bugUrl = "Muted until ml-cpp supports multiple classes")
     public void testDependentVariableCardinalityTooHighError() throws Exception {
         initialize("cardinality_too_high");
         indexData(sourceIndex, 6, 5, KEYWORD_FIELD);
-
         // Index one more document with a class different than the two already used.
         client().execute(IndexAction.INSTANCE, new IndexRequest(sourceIndex)
             .source(KEYWORD_FIELD, "fox")
@@ -420,6 +435,7 @@ public void testDependentVariableIsAliasToNested() throws Exception {
         assertEvaluation(ALIAS_TO_NESTED_FIELD, KEYWORD_FIELD_VALUES, "ml." + predictedClassField);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53188")
     public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exception {
         String sourceIndex = "classification_two_jobs_with_same_randomize_seed_source";
         String dependentVariable = KEYWORD_FIELD;
@@ -437,11 +453,11 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
             .setGamma(1.0)
             .setEta(1.0)
             .setFeatureBagFraction(1.0)
-            .setMaxTrees(1)
+            .setMaximumNumberTrees(1)
             .build();
 
         DataFrameAnalyticsConfig firstJob = buildAnalytics(firstJobId, sourceIndex, firstJobDestIndex, null,
-            new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, null));
+            new Classification(dependentVariable, boostedTreeParams, null, 1, 50.0, null));
         registerAnalytics(firstJob);
         putAnalytics(firstJob);
 
@@ -450,7 +466,7 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
 
         long randomizeSeed = ((Classification) firstJob.getAnalysis()).getRandomizeSeed();
         DataFrameAnalyticsConfig secondJob = buildAnalytics(secondJobId, sourceIndex, secondJobDestIndex, null,
-            new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, randomizeSeed));
+            new Classification(dependentVariable, boostedTreeParams, null, 1, 50.0, randomizeSeed));
 
         registerAnalytics(secondJob);
         putAnalytics(secondJob);
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
index ca0c99eb4b421..7d1506286362b 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -26,7 +27,6 @@
 import org.elasticsearch.xpack.core.ml.job.config.Detector;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
-import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats;
@@ -92,14 +92,12 @@ public void testDeleteExpiredData_GivenNothingToDelete() throws Exception {
 
     public void testDeleteExpiredData() throws Exception {
         // Index some unused state documents (more than 10K to test scrolling works)
-        String mlStateIndexName = AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-000001";
-        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         for (int i = 0; i < 10010; i++) {
             String docId = "non_existing_job_" + randomFrom("model_state_1234567#" + i, "quantiles", "categorizer_state#" + i);
-            IndexRequest indexRequest =
-                new IndexRequest(mlStateIndexName)
-                    .id(docId)
-                    .source(Collections.emptyMap());
+            IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).id(docId);
+            indexRequest.source(Collections.emptyMap());
             bulkRequestBuilder.add(indexRequest);
         }
         ActionFuture<BulkResponse> indexUnusedStateDocsResponse = bulkRequestBuilder.execute();
@@ -164,7 +162,7 @@ public void testDeleteExpiredData() throws Exception {
         }
 
         // Refresh to ensure the snapshot timestamp updates are visible
-        refresh("*");
+        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
 
         // We need to wait a second to ensure the second time around model snapshots will have a different ID (it depends on epoch seconds)
         // FIXME it would be better to wait for something concrete instead of wait for time to elapse
@@ -295,6 +293,6 @@ private void retainAllSnapshots(String jobId) throws Exception {
             client().execute(UpdateModelSnapshotAction.INSTANCE, request).get();
         }
         // We need to refresh to ensure the updates are visible
-        refresh("*");
+        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
     }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
index d6c88fbfdb994..422cb00cdb192 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
@@ -120,7 +120,7 @@ public void testSimulate() throws IOException {
             "                \"num_top_feature_importance_values\": 2" +
             "          }},\n" +
             "          \"model_id\": \"test_classification\",\n" +
-            "          \"field_map\": {\n" +
+            "          \"field_mappings\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -133,7 +133,7 @@ public void testSimulate() throws IOException {
             "          \"target_field\": \"ml.regression\",\n" +
             "          \"model_id\": \"test_regression\",\n" +
             "          \"inference_config\": {\"regression\":{}},\n" +
-            "          \"field_map\": {\n" +
+            "          \"field_mappings\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -166,7 +166,7 @@ public void testSimulate() throws IOException {
             "        \"inference\": {\n" +
             "          \"model_id\": \"test_classification_missing\",\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
-            "          \"field_map\": {\n" +
+            "          \"field_mappings\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -191,40 +191,6 @@ public void testSimulate() throws IOException {
         assertThat(responseString, containsString("Could not find trained model [test_classification_missing]"));
     }
 
-    public void testSimulateWithDefaultMappedField() throws IOException {
-        String source = "{\n" +
-            "  \"pipeline\": {\n" +
-            "    \"processors\": [\n" +
-            "      {\n" +
-            "        \"inference\": {\n" +
-            "          \"target_field\": \"ml.classification\",\n" +
-            "          \"inference_config\": {\"classification\": " +
-            "                {\"num_top_classes\":2, " +
-            "                \"top_classes_results_field\": \"result_class_prob\"," +
-            "                \"num_top_feature_importance_values\": 2" +
-            "          }},\n" +
-            "          \"model_id\": \"test_classification\",\n" +
-            "          \"field_map\": {}\n" +
-            "        }\n" +
-            "      }\n"+
-            "    ]\n" +
-            "  },\n" +
-            "  \"docs\": [\n" +
-            "    {\"_source\": {\n" +
-            "      \"col_1_alias\": \"female\",\n" +
-            "      \"col2\": \"M\",\n" +
-            "      \"col3\": \"none\",\n" +
-            "      \"col4\": 10\n" +
-            "    }}]\n" +
-            "}";
-
-        Response response = client().performRequest(simulateRequest(source));
-        String responseString = EntityUtils.toString(response.getEntity());
-        assertThat(responseString, containsString("\"predicted_value\":\"second\""));
-        assertThat(responseString, containsString("\"col2\":0.944"));
-        assertThat(responseString, containsString("\"col1\":0.19999"));
-    }
-
     public void testSimulateLangIdent() throws IOException {
         String source = "{\n" +
             "  \"pipeline\": {\n" +
@@ -233,7 +199,7 @@ public void testSimulateLangIdent() throws IOException {
             "        \"inference\": {\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
             "          \"model_id\": \"lang_ident_model_1\",\n" +
-            "          \"field_map\": {}\n" +
+            "          \"field_mappings\": {}\n" +
             "        }\n" +
             "      }\n" +
             "    ]\n" +
@@ -559,7 +525,6 @@ protected NamedXContentRegistry xContentRegistry() {
         "{\n" +
         "  \"input\":{\"field_names\":[\"col1\",\"col2\",\"col3\",\"col4\"]}," +
         "  \"description\": \"test model for classification\",\n" +
-        "  \"default_field_map\": {\"col_1_alias\": \"col1\"},\n" +
         "  \"definition\": " + CLASSIFICATION_DEFINITION +
         "}";
 
@@ -570,7 +535,7 @@ protected NamedXContentRegistry xContentRegistry() {
         "          \"model_id\": \"test_classification\",\n" +
         "          \"tag\": \"classification\",\n" +
         "          \"inference_config\": {\"classification\": {}},\n" +
-        "          \"field_map\": {\n" +
+        "          \"field_mappings\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +
@@ -586,7 +551,7 @@ protected NamedXContentRegistry xContentRegistry() {
         "          \"model_id\": \"test_regression\",\n" +
         "          \"tag\": \"regression\",\n" +
         "          \"inference_config\": {\"regression\": {}},\n" +
-        "          \"field_map\": {\n" +
+        "          \"field_mappings\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
index f91afb7ac6e34..d8cba1c772589 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
@@ -27,7 +28,6 @@
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.transport.Netty4Plugin;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -40,7 +40,6 @@
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
-import org.elasticsearch.xpack.ilm.IndexLifecycle;
 import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 
 import java.io.IOException;
@@ -74,12 +73,7 @@ protected NamedXContentRegistry xContentRegistry() {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(
-            LocalStateMachineLearning.class,
-            Netty4Plugin.class,
-            ReindexPlugin.class,
-            // ILM is required for .ml-state template index settings
-            IndexLifecycle.class);
+        return Arrays.asList(LocalStateMachineLearning.class, Netty4Plugin.class, ReindexPlugin.class);
     }
 
     @Override
@@ -121,7 +115,6 @@ protected Settings externalClusterClientSettings() {
         builder.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
-        builder.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         builder.put(Environment.PATH_HOME_SETTING.getKey(), home);
         builder.put("xpack.security.transport.ssl.enabled", true);
@@ -156,7 +149,7 @@ protected DeleteExpiredDataAction.Response deleteExpiredData() throws Exception
             new DeleteExpiredDataAction.Request()).get();
 
         // We need to refresh to ensure the deletion is visible
-        refresh("*");
+        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
 
         return response;
     }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
index 5fc6d618da9ab..92e0137462eee 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
@@ -20,6 +21,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
 import org.junit.After;
+import org.junit.Before;
 
 import java.util.List;
 import java.util.Map;
@@ -43,9 +45,25 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase {
     private String sourceIndex;
     private String destIndex;
 
+    @Before
+    public void enableLogging() {
+        client().admin().cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder()
+                .put("logger.org.elasticsearch.xpack.ml.action", "DEBUG")
+                .put("logger.org.elasticsearch.xpack.ml.dataframe", "DEBUG"))
+            .get();
+    }
+
     @After
     public void cleanup() {
         cleanUp();
+        client().admin().cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder()
+                .putNull("logger.org.elasticsearch.xpack.ml.action")
+                .putNull("logger.org.elasticsearch.xpack.ml.dataframe"))
+            .get();
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
@@ -269,6 +287,7 @@ public void testStopAndRestart() throws Exception {
         assertMlResultsFieldMappings(destIndex, predictedClassField, "double");
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53188")
     public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exception {
         String sourceIndex = "regression_two_jobs_with_same_randomize_seed_source";
         indexData(sourceIndex, 100, 0);
@@ -281,7 +300,7 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
             .setGamma(1.0)
             .setEta(1.0)
             .setFeatureBagFraction(1.0)
-            .setMaxTrees(1)
+            .setMaximumNumberTrees(1)
             .build();
 
         DataFrameAnalyticsConfig firstJob = buildAnalytics(firstJobId, sourceIndex, firstJobDestIndex, null,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index d235982be4344..e77799ca1c0de 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -73,7 +73,6 @@
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAction;
-import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
@@ -145,7 +144,6 @@
 import org.elasticsearch.xpack.ml.action.TransportDeleteJobAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteModelSnapshotAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteTrainedModelAction;
-import org.elasticsearch.xpack.ml.action.TransportEstimateModelMemoryAction;
 import org.elasticsearch.xpack.ml.action.TransportEvaluateDataFrameAction;
 import org.elasticsearch.xpack.ml.action.TransportExplainDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.ml.action.TransportFinalizeJobExecutionAction;
@@ -283,7 +281,6 @@
 import org.elasticsearch.xpack.ml.rest.job.RestCloseJobAction;
 import org.elasticsearch.xpack.ml.rest.job.RestDeleteForecastAction;
 import org.elasticsearch.xpack.ml.rest.job.RestDeleteJobAction;
-import org.elasticsearch.xpack.ml.rest.job.RestEstimateModelMemoryAction;
 import org.elasticsearch.xpack.ml.rest.job.RestFlushJobAction;
 import org.elasticsearch.xpack.ml.rest.job.RestForecastJobAction;
 import org.elasticsearch.xpack.ml.rest.job.RestGetJobStatsAction;
@@ -736,7 +733,6 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             new RestFlushJobAction(),
             new RestValidateDetectorAction(),
             new RestValidateJobConfigAction(),
-            new RestEstimateModelMemoryAction(),
             new RestGetCategoriesAction(),
             new RestGetModelSnapshotsAction(),
             new RestRevertModelSnapshotAction(),
@@ -815,7 +811,6 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                 new ActionHandler<>(FlushJobAction.INSTANCE, TransportFlushJobAction.class),
                 new ActionHandler<>(ValidateDetectorAction.INSTANCE, TransportValidateDetectorAction.class),
                 new ActionHandler<>(ValidateJobConfigAction.INSTANCE, TransportValidateJobConfigAction.class),
-                new ActionHandler<>(EstimateModelMemoryAction.INSTANCE, TransportEstimateModelMemoryAction.class),
                 new ActionHandler<>(GetCategoriesAction.INSTANCE, TransportGetCategoriesAction.class),
                 new ActionHandler<>(GetModelSnapshotsAction.INSTANCE, TransportGetModelSnapshotsAction.class),
                 new ActionHandler<>(RevertModelSnapshotAction.INSTANCE, TransportRevertModelSnapshotAction.class),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
index 7f36db7a5b893..4f731d9804c4b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.ingest.IngestStats;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
@@ -41,7 +40,6 @@
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
-import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
@@ -115,8 +113,6 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat
                 addInferenceIngestUsage(response, inferenceUsage);
                 SearchRequestBuilder requestBuilder = client.prepareSearch(InferenceIndexConstants.INDEX_PATTERN)
                     .setSize(0)
-                    .setQuery(QueryBuilders.boolQuery()
-                        .filter(QueryBuilders.termQuery(InferenceIndexConstants.DOC_TYPE.getPreferredName(), TrainedModelConfig.NAME)))
                     .setTrackTotalHits(true);
                 ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(),
                     ClientHelper.ML_ORIGIN,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
index 83c0050035d38..ae516b4f8ce7a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
-import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.MlConfigIndex;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlStatsIndex;
@@ -39,8 +38,10 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
 
     private static final IndexTemplateConfig ANOMALY_DETECTION_RESULTS_TEMPLATE = anomalyDetectionResultsTemplate();
 
-    private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = stateTemplate(true);
-    private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE_NO_ILM = stateTemplate(false);
+    private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = new IndexTemplateConfig(
+        AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,ANOMALY_DETECTION_PATH + "state_index_template.json",
+        Version.CURRENT.id, VERSION_PATTERN,
+        Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
 
     private static final IndexTemplateConfig META_TEMPLATE = new IndexTemplateConfig(MlMetaIndex.INDEX_NAME,
         ROOT_RESOURCE_PATH + "meta_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
@@ -56,12 +57,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
         ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
         Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
 
-    private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate(true);
-    private static final IndexTemplateConfig STATS_TEMPLATE_NO_ILM = statsTemplate(false);
-
-    private static final String ML_SIZE_BASED_ILM_POLICY_NAME = "ml-size-based-ilm-policy";
-    private static final LifecyclePolicyConfig ML_SIZE_BASED_ILM_POLICY =
-        new LifecyclePolicyConfig(ML_SIZE_BASED_ILM_POLICY_NAME, ROOT_RESOURCE_PATH + "size_based_ilm_policy.json");
+    private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate();
 
     private static IndexTemplateConfig configTemplate() {
         Map<String, String> variables = new HashMap<>();
@@ -76,22 +72,6 @@ private static IndexTemplateConfig configTemplate() {
             variables);
     }
 
-    private static IndexTemplateConfig stateTemplate(boolean ilmEnabled) {
-        Map<String, String> variables = new HashMap<>();
-        variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
-        variables.put(
-            "xpack.ml.index.lifecycle.settings",
-            ilmEnabled
-                ? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" +
-                  ",\"index.lifecycle.rollover_alias\": \"" + AnomalyDetectorsIndex.jobStateIndexWriteAlias() + "\"\n"
-                : "");
-
-        return new IndexTemplateConfig(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
-            ANOMALY_DETECTION_PATH + "state_index_template.json",
-            Version.CURRENT.id, VERSION_PATTERN,
-            variables);
-    }
-
     private static IndexTemplateConfig anomalyDetectionResultsTemplate() {
         Map<String, String> variables = new HashMap<>();
         variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
@@ -103,16 +83,10 @@ private static IndexTemplateConfig anomalyDetectionResultsTemplate() {
             variables);
     }
 
-    private static IndexTemplateConfig statsTemplate(boolean ilmEnabled) {
+    private static IndexTemplateConfig statsTemplate() {
         Map<String, String> variables = new HashMap<>();
         variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
         variables.put("xpack.ml.stats.mappings", MlStatsIndex.mapping());
-        variables.put(
-            "xpack.ml.index.lifecycle.settings",
-            ilmEnabled
-                ? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" +
-                ",\"index.lifecycle.rollover_alias\": \"" + MlStatsIndex.writeAlias() + "\"\n"
-                : "");
 
         return new IndexTemplateConfig(MlStatsIndex.TEMPLATE_NAME,
             ROOT_RESOURCE_PATH + "stats_index_template.json",
@@ -120,20 +94,9 @@ private static IndexTemplateConfig statsTemplate(boolean ilmEnabled) {
             variables);
     }
 
-    private final List<IndexTemplateConfig> templatesToUse;
-
     public MlIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client,
                                    NamedXContentRegistry xContentRegistry) {
         super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
-        boolean ilmEnabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings);
-        templatesToUse = Arrays.asList(
-            ANOMALY_DETECTION_RESULTS_TEMPLATE,
-            ilmEnabled ? ANOMALY_DETECTION_STATE_TEMPLATE : ANOMALY_DETECTION_STATE_TEMPLATE_NO_ILM,
-            CONFIG_TEMPLATE,
-            INFERENCE_TEMPLATE,
-            META_TEMPLATE,
-            NOTIFICATIONS_TEMPLATE,
-            ilmEnabled ? STATS_TEMPLATE : STATS_TEMPLATE_NO_ILM);
     }
 
     @Override
@@ -143,12 +106,20 @@ protected boolean requiresMasterNode() {
 
     @Override
     protected List<IndexTemplateConfig> getTemplateConfigs() {
-        return templatesToUse;
+        return Arrays.asList(
+            ANOMALY_DETECTION_RESULTS_TEMPLATE,
+            ANOMALY_DETECTION_STATE_TEMPLATE,
+            CONFIG_TEMPLATE,
+            INFERENCE_TEMPLATE,
+            META_TEMPLATE,
+            NOTIFICATIONS_TEMPLATE,
+            STATS_TEMPLATE
+        );
     }
 
     @Override
     protected List<LifecyclePolicyConfig> getPolicyConfigs() {
-        return Collections.singletonList(ML_SIZE_BASED_ILM_POLICY);
+        return Collections.emptyList();
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
deleted file mode 100644
index d415156450082..0000000000000
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.action;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
-import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
-import org.elasticsearch.xpack.core.ml.job.config.Detector;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public class TransportEstimateModelMemoryAction
-    extends HandledTransportAction<EstimateModelMemoryAction.Request, EstimateModelMemoryAction.Response> {
-
-    private static final Logger logger = LogManager.getLogger(TransportEstimateModelMemoryAction.class);
-
-    static final ByteSizeValue BASIC_REQUIREMENT = new ByteSizeValue(10, ByteSizeUnit.MB);
-    static final long BYTES_PER_INFLUENCER_VALUE = new ByteSizeValue(10, ByteSizeUnit.KB).getBytes();
-    private static final long BYTES_IN_MB = new ByteSizeValue(1, ByteSizeUnit.MB).getBytes();
-
-    @Inject
-    public TransportEstimateModelMemoryAction(TransportService transportService,
-                                              ActionFilters actionFilters) {
-        super(EstimateModelMemoryAction.NAME, transportService, actionFilters, EstimateModelMemoryAction.Request::new);
-    }
-
-    @Override
-    protected void doExecute(Task task,
-                             EstimateModelMemoryAction.Request request,
-                             ActionListener<EstimateModelMemoryAction.Response> listener) {
-
-        AnalysisConfig analysisConfig = request.getAnalysisConfig();
-        Map<String, Long> overallCardinality = request.getOverallCardinality();
-        Map<String, Long> maxBucketCardinality = request.getMaxBucketCardinality();
-
-        long answer = BASIC_REQUIREMENT.getBytes()
-            + calculateDetectorsRequirementBytes(analysisConfig, overallCardinality)
-            + calculateInfluencerRequirementBytes(analysisConfig, maxBucketCardinality)
-            + calculateCategorizationRequirementBytes(analysisConfig);
-
-        listener.onResponse(new EstimateModelMemoryAction.Response(roundUpToNextMb(answer)));
-    }
-
-    static long calculateDetectorsRequirementBytes(AnalysisConfig analysisConfig, Map<String, Long> overallCardinality) {
-        return analysisConfig.getDetectors().stream().map(detector -> calculateDetectorRequirementBytes(detector, overallCardinality))
-            .reduce(0L, Long::sum);
-    }
-
-    static long calculateDetectorRequirementBytes(Detector detector, Map<String, Long> overallCardinality) {
-
-        long answer = 0;
-
-        switch (detector.getFunction()) {
-            case COUNT:
-            case LOW_COUNT:
-            case HIGH_COUNT:
-            case NON_ZERO_COUNT:
-            case LOW_NON_ZERO_COUNT:
-            case HIGH_NON_ZERO_COUNT:
-                answer = 1; // TODO add realistic number
-                break;
-            case DISTINCT_COUNT:
-            case LOW_DISTINCT_COUNT:
-            case HIGH_DISTINCT_COUNT:
-                answer = 1; // TODO add realistic number
-                break;
-            case RARE:
-            case FREQ_RARE:
-                answer = 1; // TODO add realistic number
-                break;
-            case INFO_CONTENT:
-            case LOW_INFO_CONTENT:
-            case HIGH_INFO_CONTENT:
-                answer = 1; // TODO add realistic number
-                break;
-            case METRIC:
-                answer = 1; // TODO add realistic number
-                break;
-            case MEAN:
-            case LOW_MEAN:
-            case HIGH_MEAN:
-            case AVG:
-            case LOW_AVG:
-            case HIGH_AVG:
-            case MIN:
-            case MAX:
-            case SUM:
-            case LOW_SUM:
-            case HIGH_SUM:
-            case NON_NULL_SUM:
-            case LOW_NON_NULL_SUM:
-            case HIGH_NON_NULL_SUM:
-                // 64 comes from https://github.com/elastic/kibana/issues/18722
-                answer = new ByteSizeValue(64, ByteSizeUnit.KB).getBytes();
-                break;
-            case MEDIAN:
-            case LOW_MEDIAN:
-            case HIGH_MEDIAN:
-                answer = 1; // TODO add realistic number
-                break;
-            case VARP:
-            case LOW_VARP:
-            case HIGH_VARP:
-                answer = 1; // TODO add realistic number
-                break;
-            case TIME_OF_DAY:
-            case TIME_OF_WEEK:
-                answer = 1; // TODO add realistic number
-                break;
-            case LAT_LONG:
-                answer = 1; // TODO add realistic number
-                break;
-            default:
-                assert false : "unhandled detector function: " + detector.getFunction().getFullName();
-        }
-
-        String byFieldName = detector.getByFieldName();
-        if (byFieldName != null) {
-            answer *= cardinalityEstimate(Detector.BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName, overallCardinality, true);
-        }
-
-        String overFieldName = detector.getOverFieldName();
-        if (overFieldName != null) {
-            cardinalityEstimate(Detector.OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName, overallCardinality, true);
-            // TODO - how should "over" field cardinality affect estimate?
-        }
-
-        String partitionFieldName = detector.getPartitionFieldName();
-        if (partitionFieldName != null) {
-            answer *=
-                cardinalityEstimate(Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName, overallCardinality, true);
-        }
-
-        return answer;
-    }
-
-    static long calculateInfluencerRequirementBytes(AnalysisConfig analysisConfig, Map<String, Long> maxBucketCardinality) {
-
-        // Influencers that are also by/over/partition fields do not consume extra memory by being influencers
-        Set<String> pureInfluencers = new HashSet<>(analysisConfig.getInfluencers());
-        for (Detector detector : analysisConfig.getDetectors()) {
-            pureInfluencers.removeAll(detector.extractAnalysisFields());
-        }
-
-        return pureInfluencers.stream()
-            .map(influencer -> cardinalityEstimate(AnalysisConfig.INFLUENCERS.getPreferredName(), influencer, maxBucketCardinality, false)
-                * BYTES_PER_INFLUENCER_VALUE)
-            .reduce(0L, Long::sum);
-    }
-
-    static long calculateCategorizationRequirementBytes(AnalysisConfig analysisConfig) {
-
-        if (analysisConfig.getCategorizationFieldName() != null) {
-            return 1; // TODO add realistic number
-        } else {
-            return 0;
-        }
-    }
-
-    static long cardinalityEstimate(String description, String fieldName, Map<String, Long> suppliedCardinailityEstimates,
-                                    boolean isOverall) {
-        Long suppliedEstimate = suppliedCardinailityEstimates.get(fieldName);
-        if (suppliedEstimate != null) {
-            return suppliedEstimate;
-        }
-        // Don't expect the user to supply cardinality estimates for the mlcategory field that we create ourselves
-        if (AnalysisConfig.ML_CATEGORY_FIELD.equals(fieldName)) {
-            return isOverall ? 500 : 50;
-        }
-        logger.warn("[{}] cardinality estimate required for [{}] [{}] but not supplied",
-            isOverall ? "Overall" : "Bucket max", description, fieldName);
-        return 0;
-    }
-
-    static ByteSizeValue roundUpToNextMb(long bytes) {
-        assert bytes >= 0;
-        return new ByteSizeValue((BYTES_IN_MB - 1 + bytes) / BYTES_IN_MB, ByteSizeUnit.MB);
-    }
-}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
index c6d92413dd392..5c48be663f117 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
@@ -19,41 +18,26 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.Evaluation;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.ml.utils.TypedChainTaskExecutor;
 
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.elasticsearch.search.aggregations.MultiBucketConsumerService.MAX_BUCKET_SETTING;
 
 public class TransportEvaluateDataFrameAction extends HandledTransportAction<EvaluateDataFrameAction.Request,
     EvaluateDataFrameAction.Response> {
 
     private final ThreadPool threadPool;
     private final Client client;
-    private final AtomicReference<Integer> maxBuckets = new AtomicReference<>();
 
     @Inject
-    public TransportEvaluateDataFrameAction(TransportService transportService,
-                                            ActionFilters actionFilters,
-                                            ThreadPool threadPool,
-                                            Client client,
-                                            ClusterService clusterService) {
+    public TransportEvaluateDataFrameAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool,
+                                            Client client) {
         super(EvaluateDataFrameAction.NAME, transportService, actionFilters, EvaluateDataFrameAction.Request::new);
         this.threadPool = threadPool;
         this.client = client;
-        this.maxBuckets.set(MAX_BUCKET_SETTING.get(clusterService.getSettings()));
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_BUCKET_SETTING, this::setMaxBuckets);
-    }
-
-    private void setMaxBuckets(int maxBuckets) {
-        this.maxBuckets.set(maxBuckets);
     }
 
     @Override
-    protected void doExecute(Task task,
-                             EvaluateDataFrameAction.Request request,
+    protected void doExecute(Task task, EvaluateDataFrameAction.Request request,
                              ActionListener<EvaluateDataFrameAction.Response> listener) {
         ActionListener<List<Void>> resultsListener = ActionListener.wrap(
             unused -> {
@@ -64,9 +48,7 @@ protected void doExecute(Task task,
             listener::onFailure
         );
 
-        // Create an immutable collection of parameters to be used by evaluation metrics.
-        EvaluationParameters parameters = new EvaluationParameters(maxBuckets.get());
-        EvaluationExecutor evaluationExecutor = new EvaluationExecutor(threadPool, client, parameters, request);
+        EvaluationExecutor evaluationExecutor = new EvaluationExecutor(threadPool, client, request);
         evaluationExecutor.execute(resultsListener);
     }
 
@@ -86,14 +68,12 @@ protected void doExecute(Task task,
     private static final class EvaluationExecutor extends TypedChainTaskExecutor<Void> {
 
         private final Client client;
-        private final EvaluationParameters parameters;
         private final EvaluateDataFrameAction.Request request;
         private final Evaluation evaluation;
 
-        EvaluationExecutor(ThreadPool threadPool, Client client, EvaluationParameters parameters, EvaluateDataFrameAction.Request request) {
+        EvaluationExecutor(ThreadPool threadPool, Client client, EvaluateDataFrameAction.Request request) {
             super(threadPool.generic(), unused -> true, unused -> true);
             this.client = client;
-            this.parameters = parameters;
             this.request = request;
             this.evaluation = request.getEvaluation();
             // Add one task only. Other tasks will be added as needed by the nextTask method itself.
@@ -102,7 +82,7 @@ private static final class EvaluationExecutor extends TypedChainTaskExecutor<Voi
 
         private TypedChainTaskExecutor.ChainTask<Void> nextTask() {
             return listener -> {
-                SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(parameters, request.getParsedQuery());
+                SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(request.getParsedQuery());
                 SearchRequest searchRequest = new SearchRequest(request.getIndices()).source(searchSourceBuilder);
                 client.execute(
                     SearchAction.INSTANCE,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
index ca80f6fd8e72c..052f3ed4ddca7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
@@ -7,7 +7,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
@@ -98,7 +97,6 @@ public class TransportStartDataFrameAnalyticsAction
     extends TransportMasterNodeAction<StartDataFrameAnalyticsAction.Request, AcknowledgedResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportStartDataFrameAnalyticsAction.class);
-    private static final String PRIMARY_SHARDS_INACTIVE = "not all primary shards are active";
 
     private final XPackLicenseState licenseState;
     private final Client client;
@@ -412,26 +410,8 @@ public void onFailure(Exception e) {
 
                 @Override
                 public void onTimeout(TimeValue timeout) {
-                    logger.error(
-                        () -> new ParameterizedMessage("[{}] timed out when starting task after [{}]. Assignment explanation [{}]",
-                            task.getParams().getId(),
-                            timeout,
-                            predicate.assignmentExplanation));
-                    if (predicate.assignmentExplanation != null) {
-                        cancelAnalyticsStart(task,
-                            new ElasticsearchStatusException(
-                                "Could not start data frame analytics task, timed out after [{}] waiting for task assignment. "
-                                    + "Assignment explanation [{}]",
-                                RestStatus.TOO_MANY_REQUESTS,
-                                timeout,
-                                predicate.assignmentExplanation),
-                            listener);
-                    } else {
-                        listener.onFailure(new ElasticsearchException(
-                            "Starting data frame analytics [{}] timed out after [{}]",
-                            task.getParams().getId(),
-                            timeout));
-                    }
+                    listener.onFailure(new ElasticsearchException(
+                        "Starting data frame analytics [" + task.getParams().getId() + "] timed out after [" + timeout + "]"));
                 }
         });
     }
@@ -456,7 +436,6 @@ private StartContext(DataFrameAnalyticsConfig config, List<PhaseProgress> progre
     private static class AnalyticsPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
 
         private volatile Exception exception;
-        private volatile String assignmentExplanation;
 
         @Override
         public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTask) {
@@ -471,15 +450,9 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTa
                 return true;
             }
 
-            if (assignment != null
-                && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false
-                && assignment.isAssigned() == false) {
-                assignmentExplanation = assignment.getExplanation();
-                // Assignment failed due to primary shard check.
-                // This is hopefully intermittent and we should allow another assignment attempt.
-                if (assignmentExplanation.contains(PRIMARY_SHARDS_INACTIVE)) {
-                    return false;
-                }
+            if (assignment != null && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false &&
+                assignment.isAssigned() == false) {
+                // Assignment has failed despite passing our "fast fail" validation
                 exception = new ElasticsearchStatusException("Could not start data frame analytics task, allocation explanation [" +
                     assignment.getExplanation() + "]", RestStatus.TOO_MANY_REQUESTS);
                 return true;
@@ -610,12 +583,8 @@ public PersistentTasksCustomMetaData.Assignment getAssignment(StartDataFrameAnal
                     MlStatsIndex.indexPattern(),
                     AnomalyDetectorsIndex.jobStateIndexPattern());
             if (unavailableIndices.size() != 0) {
-                String reason = "Not opening data frame analytics job ["
-                    + id
-                    + "], because "
-                    + PRIMARY_SHARDS_INACTIVE
-                    + " for the following indices ["
-                    + String.join(",", unavailableIndices) + "]";
+                String reason = "Not opening data frame analytics job [" + id +
+                    "], because not all primary shards are active for the following indices [" + String.join(",", unavailableIndices) + "]";
                 logger.debug(reason);
                 return new PersistentTasksCustomMetaData.Assignment(null, reason);
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
index eb13f2395ef8d..8202c0ef3d28e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
@@ -14,7 +14,6 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -28,7 +27,7 @@ public class TimeBasedExtractedFields extends ExtractedFields {
     private final ExtractedField timeField;
 
     public TimeBasedExtractedFields(ExtractedField timeField, List<ExtractedField> allFields) {
-        super(allFields, Collections.emptyMap());
+        super(allFields);
         if (!allFields.contains(timeField)) {
             throw new IllegalArgumentException("timeField should also be contained in allFields");
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
index ba1bdf9a6a5d0..6d9b6fb04b839 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
@@ -234,10 +234,6 @@ public List<String> getFieldNames() {
         return context.extractedFields.getAllFields().stream().map(ExtractedField::getName).collect(Collectors.toList());
     }
 
-    public List<ExtractedField> getAllExtractedFields() {
-        return context.extractedFields.getAllFields();
-    }
-
     public DataSummary collectDataSummary() {
         SearchRequestBuilder searchRequestBuilder = buildDataSummarySearchRequestBuilder();
         SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
index 2254a3de0a60c..0c885806d730c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
@@ -58,15 +58,15 @@ public class ExtractedFieldsDetector {
     private final DataFrameAnalyticsConfig config;
     private final int docValueFieldsLimit;
     private final FieldCapabilitiesResponse fieldCapabilitiesResponse;
-    private final Map<String, Long> cardinalitiesForFieldsWithConstraints;
+    private final Map<String, Long> fieldCardinalities;
 
     ExtractedFieldsDetector(String[] index, DataFrameAnalyticsConfig config, int docValueFieldsLimit,
-                            FieldCapabilitiesResponse fieldCapabilitiesResponse, Map<String, Long> cardinalitiesForFieldsWithConstraints) {
+                            FieldCapabilitiesResponse fieldCapabilitiesResponse, Map<String, Long> fieldCardinalities) {
         this.index = Objects.requireNonNull(index);
         this.config = Objects.requireNonNull(config);
         this.docValueFieldsLimit = docValueFieldsLimit;
         this.fieldCapabilitiesResponse = Objects.requireNonNull(fieldCapabilitiesResponse);
-        this.cardinalitiesForFieldsWithConstraints = Objects.requireNonNull(cardinalitiesForFieldsWithConstraints);
+        this.fieldCardinalities = Objects.requireNonNull(fieldCardinalities);
     }
 
     public Tuple<ExtractedFields, List<FieldSelection>> detect() {
@@ -286,13 +286,12 @@ private void checkRequiredFields(Set<String> fields) {
 
     private void checkFieldsWithCardinalityLimit() {
         for (FieldCardinalityConstraint constraint : config.getAnalysis().getFieldCardinalityConstraints()) {
-            constraint.check(cardinalitiesForFieldsWithConstraints.get(constraint.getField()));
+            constraint.check(fieldCardinalities.get(constraint.getField()));
         }
     }
 
     private ExtractedFields detectExtractedFields(Set<String> fields, Set<FieldSelection> fieldSelection) {
-        ExtractedFields extractedFields = ExtractedFields.build(fields, Collections.emptySet(), fieldCapabilitiesResponse,
-            cardinalitiesForFieldsWithConstraints);
+        ExtractedFields extractedFields = ExtractedFields.build(fields, Collections.emptySet(), fieldCapabilitiesResponse);
         boolean preferSource = extractedFields.getDocValueFields().size() > docValueFieldsLimit;
         extractedFields = deduplicateMultiFields(extractedFields, preferSource, fieldSelection);
         if (preferSource) {
@@ -322,7 +321,7 @@ private ExtractedFields deduplicateMultiFields(ExtractedFields extractedFields,
                     chooseMultiFieldOrParent(preferSource, requiredFields, parent, multiField, fieldSelection));
             }
         }
-        return new ExtractedFields(new ArrayList<>(nameOrParentToField.values()), cardinalitiesForFieldsWithConstraints);
+        return new ExtractedFields(new ArrayList<>(nameOrParentToField.values()));
     }
 
     private ExtractedField chooseMultiFieldOrParent(boolean preferSource, Set<String> requiredFields, ExtractedField parent,
@@ -373,7 +372,7 @@ private ExtractedFields fetchFromSourceIfSupported(ExtractedFields extractedFiel
         for (ExtractedField field : extractedFields.getAllFields()) {
             adjusted.add(field.supportsFromSource() ? field.newFromSource() : field);
         }
-        return new ExtractedFields(adjusted, cardinalitiesForFieldsWithConstraints);
+        return new ExtractedFields(adjusted);
     }
 
     private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFields) {
@@ -390,7 +389,7 @@ private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFi
                 adjusted.add(field);
             }
         }
-        return new ExtractedFields(adjusted, cardinalitiesForFieldsWithConstraints);
+        return new ExtractedFields(adjusted);
     }
 
     private void addIncludedFields(ExtractedFields extractedFields, Set<FieldSelection> fieldSelection) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
index 0daec5365ff48..714f63091801f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
@@ -14,9 +14,10 @@
 
 import java.io.IOException;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 
+import static java.util.stream.Collectors.toMap;
+
 public class AnalyticsProcessConfig implements ToXContentObject {
 
     private static final String JOB_ID = "job_id";
@@ -92,31 +93,12 @@ private DataFrameAnalysisWrapper(DataFrameAnalysis analysis, ExtractedFields ext
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("name", analysis.getWriteableName());
-            builder.field("parameters", analysis.getParams(new AnalysisFieldInfo(extractedFields)));
+            builder.field(
+                "parameters",
+                analysis.getParams(
+                    extractedFields.getAllFields().stream().collect(toMap(ExtractedField::getName, ExtractedField::getTypes))));
             builder.endObject();
             return builder;
         }
     }
-
-    private static class AnalysisFieldInfo implements DataFrameAnalysis.FieldInfo {
-
-        private final ExtractedFields extractedFields;
-
-        AnalysisFieldInfo(ExtractedFields extractedFields) {
-            this.extractedFields = Objects.requireNonNull(extractedFields);
-        }
-
-        @Override
-        public Set<String> getTypes(String field) {
-            Optional<ExtractedField> extractedField = extractedFields.getAllFields().stream()
-                .filter(f -> f.getName().equals(field))
-                .findAny();
-            return extractedField.isPresent() ? extractedField.get().getTypes() : null;
-        }
-
-        @Override
-        public Long getCardinality(String field) {
-            return extractedFields.getCardinalitiesForFieldsWithConstraints().get(field);
-        }
-    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
index 255ecfd77c021..b967e1d47118c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
@@ -434,7 +434,7 @@ private AnalyticsResultProcessor createResultProcessor(DataFrameAnalyticsTask ta
                 new DataFrameRowsJoiner(config.getId(), dataExtractorFactory.newExtractor(true), resultsPersisterService);
             return new AnalyticsResultProcessor(
                 config, dataFrameRowsJoiner, task.getStatsHolder(), trainedModelProvider, auditor, resultsPersisterService,
-                dataExtractor.get().getAllExtractedFields());
+                dataExtractor.get().getFieldNames());
         }
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
index 34be13967b5d2..dff78a1cc4daf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
@@ -33,8 +33,6 @@
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
-import org.elasticsearch.xpack.ml.extractor.ExtractedField;
-import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
@@ -44,12 +42,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
 
@@ -75,7 +71,7 @@ public class AnalyticsResultProcessor {
     private final TrainedModelProvider trainedModelProvider;
     private final DataFrameAnalyticsAuditor auditor;
     private final ResultsPersisterService resultsPersisterService;
-    private final List<ExtractedField> fieldNames;
+    private final List<String> fieldNames;
     private final CountDownLatch completionLatch = new CountDownLatch(1);
     private volatile String failure;
     private volatile boolean isCancelled;
@@ -83,7 +79,7 @@ public class AnalyticsResultProcessor {
     public AnalyticsResultProcessor(DataFrameAnalyticsConfig analytics, DataFrameRowsJoiner dataFrameRowsJoiner,
                                     StatsHolder statsHolder, TrainedModelProvider trainedModelProvider,
                                     DataFrameAnalyticsAuditor auditor, ResultsPersisterService resultsPersisterService,
-                                    List<ExtractedField> fieldNames) {
+                                    List<String> fieldNames) {
         this.analytics = Objects.requireNonNull(analytics);
         this.dataFrameRowsJoiner = Objects.requireNonNull(dataFrameRowsJoiner);
         this.statsHolder = Objects.requireNonNull(statsHolder);
@@ -197,12 +193,8 @@ private TrainedModelConfig createTrainedModelConfig(TrainedModelDefinition.Build
         TrainedModelDefinition definition = inferenceModel.build();
         String dependentVariable = getDependentVariable();
         List<String> fieldNamesWithoutDependentVariable = fieldNames.stream()
-            .map(ExtractedField::getName)
             .filter(f -> f.equals(dependentVariable) == false)
             .collect(toList());
-        Map<String, String> defaultFieldMapping = fieldNames.stream()
-            .filter(ef -> ef instanceof MultiField && (ef.getName().equals(dependentVariable) == false))
-            .collect(Collectors.toMap(ExtractedField::getParentField, ExtractedField::getName));
         return TrainedModelConfig.builder()
             .setModelId(modelId)
             .setCreatedBy(XPackUser.NAME)
@@ -218,7 +210,6 @@ private TrainedModelConfig createTrainedModelConfig(TrainedModelDefinition.Build
             .setParsedDefinition(inferenceModel)
             .setInput(new TrainedModelInput(fieldNamesWithoutDependentVariable))
             .setLicenseLevel(License.OperationMode.PLATINUM.description())
-            .setDefaultFieldMap(defaultFieldMapping)
             .build();
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
index ab314a5d21851..3a36bb7ff76d0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
@@ -28,14 +28,12 @@ public class ExtractedFields {
     private final List<ExtractedField> allFields;
     private final List<ExtractedField> docValueFields;
     private final String[] sourceFields;
-    private final Map<String, Long> cardinalitiesForFieldsWithConstraints;
 
-    public ExtractedFields(List<ExtractedField> allFields, Map<String, Long> cardinalitiesForFieldsWithConstraints) {
+    public ExtractedFields(List<ExtractedField> allFields) {
         this.allFields = Collections.unmodifiableList(allFields);
         this.docValueFields = filterFields(ExtractedField.Method.DOC_VALUE, allFields);
         this.sourceFields = filterFields(ExtractedField.Method.SOURCE, allFields).stream().map(ExtractedField::getSearchField)
             .toArray(String[]::new);
-        this.cardinalitiesForFieldsWithConstraints = Collections.unmodifiableMap(cardinalitiesForFieldsWithConstraints);
     }
 
     public List<ExtractedField> getAllFields() {
@@ -50,20 +48,14 @@ public List<ExtractedField> getDocValueFields() {
         return docValueFields;
     }
 
-    public Map<String, Long> getCardinalitiesForFieldsWithConstraints() {
-        return cardinalitiesForFieldsWithConstraints;
-    }
-
     private static List<ExtractedField> filterFields(ExtractedField.Method method, List<ExtractedField> fields) {
         return fields.stream().filter(field -> field.getMethod() == method).collect(Collectors.toList());
     }
 
     public static ExtractedFields build(Collection<String> allFields, Set<String> scriptFields,
-                                        FieldCapabilitiesResponse fieldsCapabilities,
-                                        Map<String, Long> cardinalitiesForFieldsWithConstraints) {
+                                        FieldCapabilitiesResponse fieldsCapabilities) {
         ExtractionMethodDetector extractionMethodDetector = new ExtractionMethodDetector(scriptFields, fieldsCapabilities);
-        return new ExtractedFields(allFields.stream().map(field -> extractionMethodDetector.detect(field)).collect(Collectors.toList()),
-            cardinalitiesForFieldsWithConstraints);
+        return new ExtractedFields(allFields.stream().map(field -> extractionMethodDetector.detect(field)).collect(Collectors.toList()));
     }
 
     public static TimeField newTimeField(String name, ExtractedField.Method method) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java
index d856022c09d69..e41905a5faddd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java
@@ -17,7 +17,7 @@ public class MultiField implements ExtractedField {
     private final ExtractedField field;
     private final String parent;
 
-    public MultiField(String parent, ExtractedField field) {
+    MultiField(String parent, ExtractedField field) {
         this(field.getName(), field.getSearchField(), parent, field);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index 2a4861c5247f7..d36aa211b1e9d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -35,7 +34,7 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.ml.inference.loadingservice.Model;
+import org.elasticsearch.xpack.core.ml.utils.MapHelper;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
 
 import java.util.Arrays;
@@ -64,7 +63,6 @@ public class InferenceProcessor extends AbstractProcessor {
     public static final String INFERENCE_CONFIG = "inference_config";
     public static final String TARGET_FIELD = "target_field";
     public static final String FIELD_MAPPINGS = "field_mappings";
-    public static final String FIELD_MAP = "field_map";
     private static final String DEFAULT_TARGET_FIELD = "ml.inference";
 
     private final Client client;
@@ -72,7 +70,7 @@ public class InferenceProcessor extends AbstractProcessor {
 
     private final String targetField;
     private final InferenceConfig inferenceConfig;
-    private final Map<String, String> fieldMap;
+    private final Map<String, String> fieldMapping;
     private final InferenceAuditor auditor;
     private volatile boolean previouslyLicensed;
     private final AtomicBoolean shouldAudit = new AtomicBoolean(true);
@@ -83,14 +81,14 @@ public InferenceProcessor(Client client,
                               String targetField,
                               String modelId,
                               InferenceConfig inferenceConfig,
-                              Map<String, String> fieldMap) {
+                              Map<String, String> fieldMapping) {
         super(tag);
         this.client = ExceptionsHelper.requireNonNull(client, "client");
         this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD);
         this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor");
         this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
         this.inferenceConfig = ExceptionsHelper.requireNonNull(inferenceConfig, INFERENCE_CONFIG);
-        this.fieldMap = ExceptionsHelper.requireNonNull(fieldMap, FIELD_MAP);
+        this.fieldMapping = ExceptionsHelper.requireNonNull(fieldMapping, FIELD_MAPPINGS);
     }
 
     public String getModelId() {
@@ -128,7 +126,14 @@ void handleResponse(InternalInferModelAction.Response response,
 
     InternalInferModelAction.Request buildRequest(IngestDocument ingestDocument) {
         Map<String, Object> fields = new HashMap<>(ingestDocument.getSourceAndMetadata());
-        Model.mapFieldsIfNecessary(fields, fieldMap);
+        if (fieldMapping != null) {
+            fieldMapping.forEach((src, dest) -> {
+                Object srcValue = MapHelper.dig(src, fields);
+                if (srcValue != null) {
+                    fields.put(dest, srcValue);
+                }
+            });
+        }
         return new InternalInferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed);
     }
 
@@ -237,14 +242,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
             // If multiple inference processors are in the same pipeline, it is wise to tag them
             // The tag will keep default value entries from stepping on each other
             String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, TARGET_FIELD, defaultTargetField);
-            Map<String, String> fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAP);
-            if (fieldMap == null) {
-                fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
-                //TODO Remove in 8.x
-                if (fieldMap != null) {
-                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
-                }
-            }
+            Map<String, String> fieldMapping = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
             InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
 
             return new InferenceProcessor(client,
@@ -253,7 +251,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
                 targetField,
                 modelId,
                 inferenceConfig,
-                fieldMap);
+                fieldMapping);
         }
 
         // Package private for testing
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java
index f77188853ad96..8a233d534a76f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
 import org.elasticsearch.xpack.core.ml.utils.MapHelper;
 
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -29,16 +28,11 @@ public class LocalModel implements Model {
     private final TrainedModelDefinition trainedModelDefinition;
     private final String modelId;
     private final Set<String> fieldNames;
-    private final Map<String, String> defaultFieldMap;
 
-    public LocalModel(String modelId,
-                      TrainedModelDefinition trainedModelDefinition,
-                      TrainedModelInput input,
-                      Map<String, String> defaultFieldMap) {
+    public LocalModel(String modelId, TrainedModelDefinition trainedModelDefinition, TrainedModelInput input) {
         this.trainedModelDefinition = trainedModelDefinition;
         this.modelId = modelId;
         this.fieldNames = new HashSet<>(input.getFieldNames());
-        this.defaultFieldMap = defaultFieldMap == null ? null : new HashMap<>(defaultFieldMap);
     }
 
     long ramBytesUsed() {
@@ -67,7 +61,6 @@ public String getResultsType() {
     @Override
     public void infer(Map<String, Object> fields, InferenceConfig config, ActionListener<InferenceResults> listener) {
         try {
-            Model.mapFieldsIfNecessary(fields, defaultFieldMap);
             if (fieldNames.stream().allMatch(f -> MapHelper.dig(f, fields) == null)) {
                 listener.onResponse(new WarningInferenceResults(Messages.getMessage(INFERENCE_WARNING_ALL_FIELDS_MISSING, modelId)));
                 return;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java
index f1df6908d6329..fb32ce7f646df 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java
@@ -8,7 +8,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig;
-import org.elasticsearch.xpack.core.ml.utils.MapHelper;
 
 import java.util.Map;
 
@@ -19,27 +18,4 @@ public interface Model {
     void infer(Map<String, Object> fields, InferenceConfig inferenceConfig, ActionListener<InferenceResults> listener);
 
     String getModelId();
-
-    /**
-     * Used for translating field names in according to the passed `fieldMappings` parameter.
-     *
-     * This mutates the `fields` parameter in-place.
-     *
-     * Fields are only appended. If the expected field name already exists, it is not created/overwritten.
-     *
-     * Original fields are not deleted.
-     *
-     * @param fields Fields to map against
-     * @param fieldMapping Field originalName to expectedName string mapping
-     */
-    static void mapFieldsIfNecessary(Map<String, Object> fields, Map<String, String> fieldMapping) {
-        if (fieldMapping != null) {
-            fieldMapping.forEach((src, dest) -> {
-                Object srcValue = MapHelper.dig(src, fields);
-                if (srcValue != null) {
-                    fields.putIfAbsent(dest, srcValue);
-                }
-            });
-        }
-    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
index 0082549f088e3..b5862acfefc27 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
@@ -142,8 +142,7 @@ public void getModel(String modelId, ActionListener<Model> modelActionListener)
                     modelActionListener.onResponse(new LocalModel(
                         trainedModelConfig.getModelId(),
                         trainedModelConfig.ensureParsedDefinition(namedXContentRegistry).getModelDefinition(),
-                        trainedModelConfig.getInput(),
-                        trainedModelConfig.getDefaultFieldMap())),
+                        trainedModelConfig.getInput())),
                 modelActionListener::onFailure
             ));
         } else {
@@ -201,8 +200,7 @@ private void handleLoadSuccess(String modelId, TrainedModelConfig trainedModelCo
         LocalModel loadedModel = new LocalModel(
             trainedModelConfig.getModelId(),
             trainedModelConfig.ensureParsedDefinition(namedXContentRegistry).getModelDefinition(),
-            trainedModelConfig.getInput(),
-            trainedModelConfig.getDefaultFieldMap());
+            trainedModelConfig.getInput());
         synchronized (loadingListeners) {
             listeners = loadingListeners.remove(modelId);
             // If there is no loadingListener that means the loading was canceled and the listener was already notified as such
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java
deleted file mode 100644
index 319446e9fd169..0000000000000
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.rest.job;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
-import org.elasticsearch.xpack.ml.MachineLearning;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-
-public class RestEstimateModelMemoryAction extends BaseRestHandler {
-
-    @Override
-    public List<Route> routes() {
-        return Collections.singletonList(new Route(POST, MachineLearning.BASE_PATH + "anomaly_detectors/_estimate_model_memory"));
-    }
-
-    @Override
-    public String getName() {
-        return "ml_estimate_model_memory_action";
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        EstimateModelMemoryAction.Request request =
-            EstimateModelMemoryAction.Request.parseRequest(restRequest.contentOrSourceParamParser());
-        return channel -> client.execute(EstimateModelMemoryAction.INSTANCE, request, new RestToXContentListener<>(channel));
-    }
-}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
index dfbc4be8e8265..0925d252e2019 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
@@ -484,7 +484,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() throws Excep
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"modelprocessorlicensetest\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_map\": {}\n" +
+            "          \"field_mappings\": {}\n" +
             "        }\n" +
             "      }]}\n";
         // Creating a pipeline should work
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java
deleted file mode 100644
index d339c54815a76..0000000000000
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.IndicesAdminClient;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterModule;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.ilm.LifecycleAction;
-import org.elasticsearch.xpack.core.ilm.RolloverAction;
-import org.elasticsearch.xpack.core.ml.MlStatsIndex;
-import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
-import org.junit.Before;
-import org.mockito.ArgumentCaptor;
-import org.mockito.stubbing.Answer;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.elasticsearch.mock.orig.Mockito.verify;
-import static org.elasticsearch.mock.orig.Mockito.when;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-
-public class MlIndexTemplateRegistryTests extends ESTestCase {
-
-    private final DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
-    private final DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
-
-    private NamedXContentRegistry xContentRegistry;
-    private ClusterService clusterService;
-    private ThreadPool threadPool;
-    private Client client;
-    private ArgumentCaptor<PutIndexTemplateRequest> putIndexTemplateRequestCaptor;
-
-    @Before
-    public void setUpMocks() {
-        threadPool = mock(ThreadPool.class);
-        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
-        when(threadPool.generic()).thenReturn(EsExecutors.newDirectExecutorService());
-
-        client = mock(Client.class);
-        when(client.threadPool()).thenReturn(threadPool);
-        AdminClient adminClient = mock(AdminClient.class);
-        IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class);
-        when(adminClient.indices()).thenReturn(indicesAdminClient);
-        when(client.admin()).thenReturn(adminClient);
-        doAnswer(withResponse(new AcknowledgedResponse(true))).when(indicesAdminClient).putTemplate(any(), any());
-
-        clusterService = mock(ClusterService.class);
-
-        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
-        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse));
-        xContentRegistry = new NamedXContentRegistry(entries);
-
-        putIndexTemplateRequestCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
-    }
-
-    public void testStateTemplateWithIlm() {
-        MlIndexTemplateRegistry registry =
-            new MlIndexTemplateRegistry(
-                Settings.builder()
-                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true)
-                    .build(),
-                clusterService, threadPool, client, xContentRegistry);
-
-        registry.clusterChanged(createClusterChangedEvent(nodes));
-
-        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
-
-        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
-            .filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX))
-            .findFirst()
-            .orElseThrow(() -> new AssertionError("expected the ml state index template to be put"));
-        assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy"));
-        assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-state-write"));
-    }
-
-    public void testStateTemplateWithNoIlm() {
-        MlIndexTemplateRegistry registry =
-            new MlIndexTemplateRegistry(
-                Settings.builder()
-                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false)
-                    .build(),
-                clusterService, threadPool, client, xContentRegistry);
-
-        registry.clusterChanged(createClusterChangedEvent(nodes));
-
-        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
-
-        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
-            .filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX))
-            .findFirst()
-            .orElseThrow(() -> new AssertionError("expected the ml state index template to be put"));
-        assertThat(req.settings().get("index.lifecycle.name"), is(nullValue()));
-        assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue()));
-    }
-
-    public void testStatsTemplateWithIlm() {
-        MlIndexTemplateRegistry registry =
-            new MlIndexTemplateRegistry(
-                Settings.builder()
-                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true)
-                    .build(),
-                clusterService, threadPool, client, xContentRegistry);
-
-        registry.clusterChanged(createClusterChangedEvent(nodes));
-
-        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
-
-        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
-            .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME))
-            .findFirst()
-            .orElseThrow(() -> new AssertionError("expected the ml stats index template to be put"));
-        assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy"));
-        assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-stats-write"));
-    }
-
-    public void testStatsTemplateWithNoIlm() {
-        MlIndexTemplateRegistry registry =
-            new MlIndexTemplateRegistry(
-                Settings.builder()
-                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false)
-                    .build(),
-                clusterService, threadPool, client, xContentRegistry);
-
-        registry.clusterChanged(createClusterChangedEvent(nodes));
-
-        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
-
-        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
-            .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME))
-            .findFirst()
-            .orElseThrow(() -> new AssertionError("expected the ml stats index template to be put"));
-        assertThat(req.settings().get("index.lifecycle.name"), is(nullValue()));
-        assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue()));
-    }
-
-    @SuppressWarnings("unchecked")
-    private static <Response> Answer<Response> withResponse(Response response) {
-        return invocationOnMock -> {
-            ActionListener<Response> listener = (ActionListener<Response>) invocationOnMock.getArguments()[1];
-            listener.onResponse(response);
-            return null;
-        };
-    }
-
-    private static ClusterChangedEvent createClusterChangedEvent(DiscoveryNodes nodes) {
-        return new ClusterChangedEvent(
-            "created-from-test",
-            ClusterState.builder(new ClusterName("test")).nodes(nodes).build(),
-            ClusterState.builder(new ClusterName("test")).build());
-    }
-}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
index 3d08d8626b5ce..734533cac54b0 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
@@ -15,9 +15,7 @@
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
-import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -46,8 +44,6 @@ protected Settings nodeSettings() {
         newSettings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
         newSettings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         newSettings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
-        // Disable ILM history index so that the tests don't have to clean it up
-        newSettings.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
         return newSettings.build();
     }
 
@@ -59,10 +55,7 @@ protected NamedXContentRegistry xContentRegistry() {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(
-            LocalStateMachineLearning.class,
-            // ILM is required for .ml-state template index settings
-            IndexLifecycle.class);
+        return pluginList(LocalStateMachineLearning.class);
     }
 
     /**
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
deleted file mode 100644
index ea10c9fb5f64c..0000000000000
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.action;
-
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
-import org.elasticsearch.xpack.core.ml.job.config.Detector;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
-public class TransportEstimateModelMemoryActionTests extends ESTestCase {
-
-    public void testCalculateDetectorRequirementBytes() {
-
-        Map<String, Long> overallCardinality = new HashMap<>();
-        overallCardinality.put("part", 100L);
-        overallCardinality.put("buy", 200L);
-        overallCardinality.put("ovr", 300L);
-
-        String function = randomFrom("mean", "min", "max", "sum");
-
-        Detector noSplit = createDetector(function, "field", null, null, null);
-        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(noSplit,
-            overallCardinality), is(65536L));
-
-        Detector withByField = createDetector(function, "field", "buy", null, null);
-        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withByField,
-            overallCardinality), is(200 * 65536L));
-
-        Detector withPartitionField = createDetector(function, "field", null, null, "part");
-        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withPartitionField,
-            overallCardinality), is(100 * 65536L));
-
-        Detector withByAndPartitionFields = createDetector(function, "field", "buy", null, "part");
-        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withByAndPartitionFields,
-            overallCardinality), is(200 * 100 * 65536L));
-    }
-
-    public void testCalculateInfluencerRequirementBytes() {
-
-        Map<String, Long> maxBucketCardinality = new HashMap<>();
-        maxBucketCardinality.put("part", 100L);
-        maxBucketCardinality.put("inf1", 200L);
-        maxBucketCardinality.put("inf2", 300L);
-
-        AnalysisConfig noInfluencers = createCountAnalysisConfig(null, null);
-        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(noInfluencers,
-            maxBucketCardinality), is(0L));
-
-        AnalysisConfig influencerAlsoPartitionField = createCountAnalysisConfig(null, "part", "part");
-        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(influencerAlsoPartitionField,
-            maxBucketCardinality), is(0L));
-
-        AnalysisConfig influencerNotPartitionField = createCountAnalysisConfig(null, "part", "inf1");
-        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(influencerNotPartitionField,
-            maxBucketCardinality), is(200 * TransportEstimateModelMemoryAction.BYTES_PER_INFLUENCER_VALUE));
-
-        AnalysisConfig otherInfluencerAsWellAsPartitionField = createCountAnalysisConfig(null, "part", "part", "inf1");
-        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(otherInfluencerAsWellAsPartitionField,
-            maxBucketCardinality), is(200 * TransportEstimateModelMemoryAction.BYTES_PER_INFLUENCER_VALUE));
-
-        AnalysisConfig twoInfluencersNotPartitionField = createCountAnalysisConfig(null, "part", "part", "inf1", "inf2");
-        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(twoInfluencersNotPartitionField,
-            maxBucketCardinality), is((200 + 300) * TransportEstimateModelMemoryAction.BYTES_PER_INFLUENCER_VALUE));
-    }
-
-    public void testCalculateCategorizationRequirementBytes() {
-
-        AnalysisConfig analysisConfigWithoutCategorization = createCountAnalysisConfig(null, null);
-        assertThat(TransportEstimateModelMemoryAction.calculateCategorizationRequirementBytes(analysisConfigWithoutCategorization), is(0L));
-
-        AnalysisConfig analysisConfigWithCategorization = createCountAnalysisConfig(randomAlphaOfLength(10), null);
-        assertThat(TransportEstimateModelMemoryAction.calculateCategorizationRequirementBytes(analysisConfigWithCategorization), is(1L));
-    }
-
-    public void testRoundUpToNextMb() {
-
-        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(0),
-            equalTo(new ByteSizeValue(0, ByteSizeUnit.BYTES)));
-        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(1),
-            equalTo(new ByteSizeValue(1, ByteSizeUnit.MB)));
-        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(randomIntBetween(1, 1024 * 1024)),
-            equalTo(new ByteSizeValue(1, ByteSizeUnit.MB)));
-        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(1024 * 1024),
-            equalTo(new ByteSizeValue(1, ByteSizeUnit.MB)));
-        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(1024 * 1024 + 1),
-            equalTo(new ByteSizeValue(2, ByteSizeUnit.MB)));
-        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(2 * 1024 * 1024),
-            equalTo(new ByteSizeValue(2, ByteSizeUnit.MB)));
-    }
-
-    public static Detector createDetector(String function, String fieldName, String byFieldName,
-                                          String overFieldName, String partitionFieldName) {
-
-        Detector.Builder detectorBuilder = new Detector.Builder(function, fieldName);
-        detectorBuilder.setByFieldName(byFieldName);
-        detectorBuilder.setOverFieldName(overFieldName);
-        detectorBuilder.setPartitionFieldName(partitionFieldName);
-        return detectorBuilder.build();
-    }
-
-    public static AnalysisConfig createCountAnalysisConfig(String categorizationFieldName, String partitionFieldName,
-                                                           String... influencerFieldNames) {
-
-        Detector.Builder detectorBuilder = new Detector.Builder("count", null);
-        detectorBuilder.setPartitionFieldName((categorizationFieldName != null) ? AnalysisConfig.ML_CATEGORY_FIELD : partitionFieldName);
-
-        AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(detectorBuilder.build()));
-
-        if (categorizationFieldName != null) {
-            builder.setCategorizationFieldName(categorizationFieldName);
-        }
-
-        if (influencerFieldNames.length > 0) {
-            builder.setInfluencers(Arrays.asList(influencerFieldNames));
-        }
-
-        return builder.build();
-    }
-}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
index d776ffd9818d9..03a011c717292 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
@@ -267,7 +267,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
                     new HashMap<String, Object>() {{
                         put(InferenceProcessor.MODEL_ID, modelId);
                         put("inference_config", Collections.singletonMap("regression", Collections.emptyMap()));
-                        put("field_map", Collections.emptyMap());
+                        put("field_mappings", Collections.emptyMap());
                         put("target_field", randomAlphaOfLength(10));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId + num,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
index b75392e03c2bb..d54997f1dbdc3 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
@@ -81,7 +81,7 @@ public void setUpTests() {
         query = QueryBuilders.matchAllQuery();
         extractedFields = new ExtractedFields(Arrays.asList(
             new DocValueField("field_1", Collections.singleton("keyword")),
-            new DocValueField("field_2", Collections.singleton("keyword"))), Collections.emptyMap());
+            new DocValueField("field_2", Collections.singleton("keyword"))));
         scrollSize = 1000;
         headers = Collections.emptyMap();
 
@@ -299,7 +299,7 @@ public void testIncludeSourceIsFalseAndAtLeastOneSourceField() throws IOExceptio
         // Explicit cast of ExtractedField args necessary for Eclipse due to https://bugs.eclipse.org/bugs/show_bug.cgi?id=530915
         extractedFields = new ExtractedFields(Arrays.asList(
             (ExtractedField) new DocValueField("field_1", Collections.singleton("keyword")),
-            (ExtractedField) new SourceField("field_2", Collections.singleton("text"))), Collections.emptyMap());
+            (ExtractedField) new SourceField("field_2", Collections.singleton("text"))));
 
         TestExtractor dataExtractor = createExtractor(false, false);
 
@@ -404,7 +404,7 @@ public void testGetCategoricalFields() {
             (ExtractedField) new DocValueField("field_integer", Collections.singleton("integer")),
             (ExtractedField) new DocValueField("field_long", Collections.singleton("long")),
             (ExtractedField) new DocValueField("field_keyword", Collections.singleton("keyword")),
-            (ExtractedField) new SourceField("field_text", Collections.singleton("text"))), Collections.emptyMap());
+            (ExtractedField) new SourceField("field_text", Collections.singleton("text"))));
         TestExtractor dataExtractor = createExtractor(true, true);
 
         assertThat(dataExtractor.getCategoricalFields(OutlierDetectionTests.createRandom()), empty());
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
index a7a5784c452f5..49a302a498b82 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
@@ -294,10 +294,10 @@ public void testDetect_GivenClassificationAndDependentVariableHasInvalidCardinal
             .build();
 
         ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(SOURCE_INDEX,
-            buildClassificationConfig("some_keyword"), 100, fieldCapabilities, Collections.singletonMap("some_keyword", 31L));
+            buildClassificationConfig("some_keyword"), 100, fieldCapabilities, Collections.singletonMap("some_keyword", 3L));
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect);
 
-        assertThat(e.getMessage(), equalTo("Field [some_keyword] must have at most [30] distinct values but there were at least [31]"));
+        assertThat(e.getMessage(), equalTo("Field [some_keyword] must have at most [2] distinct values but there were at least [3]"));
     }
 
     public void testDetect_GivenIgnoredField() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
deleted file mode 100644
index a4db8de032af5..0000000000000
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.dataframe.process;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
-import org.elasticsearch.xpack.ml.extractor.DocValueField;
-import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.hasKey;
-
-public class AnalyticsProcessConfigTests extends ESTestCase {
-
-    private String jobId;
-    private long rows;
-    private int cols;
-    private ByteSizeValue memoryLimit;
-    private int threads;
-    private String resultsField;
-    private Set<String> categoricalFields;
-
-    @Before
-    public void setUpConfigParams() {
-        jobId = randomAlphaOfLength(10);
-        rows = randomNonNegativeLong();
-        cols = randomIntBetween(1, 42000);
-        memoryLimit = new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES);
-        threads = randomIntBetween(1, 8);
-        resultsField = randomAlphaOfLength(10);
-
-        int categoricalFieldsSize = randomIntBetween(0, 5);
-        categoricalFields = new HashSet<>();
-        for (int i = 0; i < categoricalFieldsSize; i++) {
-            categoricalFields.add(randomAlphaOfLength(10));
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testToXContent_GivenOutlierDetection() throws IOException {
-        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
-            new DocValueField("field_1", Collections.singleton("double")),
-            new DocValueField("field_2", Collections.singleton("float"))), Collections.emptyMap());
-        DataFrameAnalysis analysis = new OutlierDetection.Builder().build();
-
-        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
-        Map<String, Object> asMap = toMap(processConfig);
-
-        assertRandomizedFields(asMap);
-
-        assertThat(asMap, hasKey("analysis"));
-        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
-        assertThat(analysisAsMap, hasEntry("name", "outlier_detection"));
-        assertThat(analysisAsMap, hasKey("parameters"));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testToXContent_GivenRegression() throws IOException {
-        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
-            new DocValueField("field_1", Collections.singleton("double")),
-            new DocValueField("field_2", Collections.singleton("float")),
-            new DocValueField("test_dep_var", Collections.singleton("keyword"))), Collections.emptyMap());
-        DataFrameAnalysis analysis = new Regression("test_dep_var");
-
-        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
-        Map<String, Object> asMap = toMap(processConfig);
-
-        assertRandomizedFields(asMap);
-
-        assertThat(asMap, hasKey("analysis"));
-        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
-        assertThat(analysisAsMap, hasEntry("name", "regression"));
-        assertThat(analysisAsMap, hasKey("parameters"));
-        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
-        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testToXContent_GivenClassificationAndDepVarIsKeyword() throws IOException {
-        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
-            new DocValueField("field_1", Collections.singleton("double")),
-            new DocValueField("field_2", Collections.singleton("float")),
-            new DocValueField("test_dep_var", Collections.singleton("keyword"))), Collections.singletonMap("test_dep_var", 5L));
-        DataFrameAnalysis analysis = new Classification("test_dep_var");
-
-        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
-        Map<String, Object> asMap = toMap(processConfig);
-
-        assertRandomizedFields(asMap);
-
-        assertThat(asMap, hasKey("analysis"));
-        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
-        assertThat(analysisAsMap, hasEntry("name", "classification"));
-        assertThat(analysisAsMap, hasKey("parameters"));
-        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
-        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
-        assertThat(paramsAsMap, hasEntry("prediction_field_type", "string"));
-        assertThat(paramsAsMap, hasEntry("num_classes", 5));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testToXContent_GivenClassificationAndDepVarIsInteger() throws IOException {
-        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
-            new DocValueField("field_1", Collections.singleton("double")),
-            new DocValueField("field_2", Collections.singleton("float")),
-            new DocValueField("test_dep_var", Collections.singleton("integer"))), Collections.singletonMap("test_dep_var", 8L));
-        DataFrameAnalysis analysis = new Classification("test_dep_var");
-
-        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
-        Map<String, Object> asMap = toMap(processConfig);
-
-        assertRandomizedFields(asMap);
-
-        assertThat(asMap, hasKey("analysis"));
-        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
-        assertThat(analysisAsMap, hasEntry("name", "classification"));
-        assertThat(analysisAsMap, hasKey("parameters"));
-        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
-        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
-        assertThat(paramsAsMap, hasEntry("prediction_field_type", "int"));
-        assertThat(paramsAsMap, hasEntry("num_classes", 8));
-    }
-
-    private AnalyticsProcessConfig createProcessConfig(DataFrameAnalysis analysis, ExtractedFields extractedFields) {
-        return new AnalyticsProcessConfig(jobId, rows, cols, memoryLimit, threads, resultsField, categoricalFields, analysis,
-            extractedFields);
-    }
-
-    private static Map<String, Object> toMap(AnalyticsProcessConfig config) throws IOException {
-        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            config.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            return XContentHelper.convertToMap(JsonXContent.jsonXContent, Strings.toString(builder), false);
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private void assertRandomizedFields(Map<String, Object> configAsMap) {
-        assertThat(configAsMap, hasEntry("job_id", jobId));
-        assertThat(configAsMap, hasEntry("rows", rows));
-        assertThat(configAsMap, hasEntry("cols", cols));
-        assertThat(configAsMap, hasEntry("memory_limit", memoryLimit.getBytes()));
-        assertThat(configAsMap, hasEntry("threads", threads));
-        assertThat(configAsMap, hasEntry("results_field", resultsField));
-        assertThat(configAsMap, hasKey("categorical_fields"));
-        assertThat((List<String>) configAsMap.get("categorical_fields"), containsInAnyOrder(categoricalFields.toArray()));
-    }
-}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
index 29105eca8174e..6d29f1a066363 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
@@ -169,7 +169,7 @@ public void testRunJob_Ok() {
         inOrder.verify(dataExtractor).getCategoricalFields(dataFrameAnalyticsConfig.getAnalysis());
         inOrder.verify(process).isProcessAlive();
         inOrder.verify(task).getStatsHolder();
-        inOrder.verify(dataExtractor).getAllExtractedFields();
+        inOrder.verify(dataExtractor).getFieldNames();
         inOrder.verify(executorServiceForProcess, times(2)).execute(any());  // 'processData' and 'processResults' threads
         verifyNoMoreInteractions(dataExtractor, executorServiceForProcess, process, task);
     }
@@ -227,7 +227,7 @@ public void testProcessContext_StartAndStop() throws Exception {
         inOrder.verify(dataExtractor).getCategoricalFields(dataFrameAnalyticsConfig.getAnalysis());
         inOrder.verify(process).isProcessAlive();
         inOrder.verify(task).getStatsHolder();
-        inOrder.verify(dataExtractor).getAllExtractedFields();
+        inOrder.verify(dataExtractor).getFieldNames();
         // stop
         inOrder.verify(dataExtractor).cancel();
         inOrder.verify(process).kill();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
index 2305116343961..044a06efbb2ce 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
@@ -24,10 +24,7 @@
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
-import org.elasticsearch.xpack.ml.extractor.DocValueField;
-import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
-import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
@@ -36,7 +33,6 @@
 import org.mockito.InOrder;
 import org.mockito.Mockito;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -162,13 +158,10 @@ public void testProcess_GivenInferenceModelIsStoredSuccessfully() {
             return null;
         }).when(trainedModelProvider).storeTrainedModel(any(TrainedModelConfig.class), any(ActionListener.class));
 
-        List<ExtractedField> extractedFieldList = new ArrayList<>(3);
-        extractedFieldList.add(new DocValueField("foo", Collections.emptySet()));
-        extractedFieldList.add(new MultiField("bar", new DocValueField("bar.keyword", Collections.emptySet())));
-        extractedFieldList.add(new DocValueField("baz", Collections.emptySet()));
+        List<String> expectedFieldNames = Arrays.asList("foo", "bar", "baz");
         TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder();
         givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null)));
-        AnalyticsResultProcessor resultProcessor = createResultProcessor(extractedFieldList);
+        AnalyticsResultProcessor resultProcessor = createResultProcessor(expectedFieldNames);
 
         resultProcessor.process(process);
         resultProcessor.awaitForCompletion();
@@ -184,8 +177,7 @@ public void testProcess_GivenInferenceModelIsStoredSuccessfully() {
         assertThat(storedModel.getTags(), contains(JOB_ID));
         assertThat(storedModel.getDescription(), equalTo(JOB_DESCRIPTION));
         assertThat(storedModel.getModelDefinition(), equalTo(inferenceModel.build()));
-        assertThat(storedModel.getDefaultFieldMap(), equalTo(Collections.singletonMap("bar", "bar.keyword")));
-        assertThat(storedModel.getInput().getFieldNames(), equalTo(Arrays.asList("bar.keyword", "baz")));
+        assertThat(storedModel.getInput().getFieldNames(), equalTo(Arrays.asList("bar", "baz")));
         assertThat(storedModel.getEstimatedHeapMemory(), equalTo(inferenceModel.build().ramBytesUsed()));
         assertThat(storedModel.getEstimatedOperations(), equalTo(inferenceModel.build().getTrainedModel().estimatedNumOperations()));
         Map<String, Object> metadata = storedModel.getMetadata();
@@ -243,13 +235,8 @@ private AnalyticsResultProcessor createResultProcessor() {
         return createResultProcessor(Collections.emptyList());
     }
 
-    private AnalyticsResultProcessor createResultProcessor(List<ExtractedField> fieldNames) {
-        return new AnalyticsResultProcessor(analyticsConfig,
-            dataFrameRowsJoiner,
-            statsHolder,
-            trainedModelProvider,
-            auditor,
-            resultsPersisterService,
-            fieldNames);
+    private AnalyticsResultProcessor createResultProcessor(List<String> fieldNames) {
+        return new AnalyticsResultProcessor(
+            analyticsConfig, dataFrameRowsJoiner, statsHolder, trainedModelProvider, auditor, resultsPersisterService, fieldNames);
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
index a51eafd1d8b3d..5ac983e7d505b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
@@ -32,7 +32,7 @@ public void testAllTypesOfFields() {
         ExtractedField sourceField1 = new SourceField("src1", Collections.singleton("text"));
         ExtractedField sourceField2 = new SourceField("src2", Collections.singleton("text"));
         ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
-                docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2), Collections.emptyMap());
+                docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2));
 
         assertThat(extractedFields.getAllFields().size(), equalTo(6));
         assertThat(extractedFields.getDocValueFields().stream().map(ExtractedField::getName).toArray(String[]::new),
@@ -54,7 +54,7 @@ public void testBuildGivenMixtureOfTypes() {
         when(fieldCapabilitiesResponse.getField("airline")).thenReturn(airlineCaps);
 
         ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("time", "value", "airline", "airport"),
-            new HashSet<>(Collections.singletonList("airport")), fieldCapabilitiesResponse, Collections.emptyMap());
+            new HashSet<>(Collections.singletonList("airport")), fieldCapabilitiesResponse);
 
         assertThat(extractedFields.getDocValueFields().size(), equalTo(2));
         assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("time"));
@@ -77,7 +77,7 @@ public void testBuildGivenMultiFields() {
         when(fieldCapabilitiesResponse.getField("airport.keyword")).thenReturn(keyword);
 
         ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("airline.text", "airport.keyword"),
-                Collections.emptySet(), fieldCapabilitiesResponse, Collections.emptyMap());
+                Collections.emptySet(), fieldCapabilitiesResponse);
 
         assertThat(extractedFields.getDocValueFields().size(), equalTo(1));
         assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("airport.keyword"));
@@ -119,7 +119,7 @@ public void testBuildGivenFieldWithoutMappings() {
         FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ExtractedFields.build(
-                Collections.singletonList("value"), Collections.emptySet(), fieldCapabilitiesResponse, Collections.emptyMap()));
+                Collections.singletonList("value"), Collections.emptySet(), fieldCapabilitiesResponse));
         assertThat(e.getMessage(), equalTo("cannot retrieve field [value] because it has no mappings"));
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
index cb0519d845b19..9ed05e56c2c30 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
@@ -131,7 +131,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             ingestService);
 
         Map<String, Object> config = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("unknown_type", Collections.emptyMap()));
@@ -143,7 +143,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]"));
 
         Map<String, Object> config2 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom"));
@@ -154,7 +154,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             equalTo("inference_config must be an object with one inference type mapped to an object."));
 
         Map<String, Object> config3 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap());
@@ -173,7 +173,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
         processorFactory.accept(builderClusterStateWithModelReferences(Version.V_7_5_0, "model1"));
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -191,7 +191,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -216,7 +216,7 @@ public void testCreateProcessor() {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -230,7 +230,7 @@ public void testCreateProcessor() {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -251,7 +251,7 @@ public void testCreateProcessorWithDuplicateFields() {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "ml");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(),
@@ -305,7 +305,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
                         put(InferenceProcessor.INFERENCE_CONFIG,
                                 Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap()));
                         put(InferenceProcessor.TARGET_FIELD, "new_field");
-                        put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
+                        put(InferenceProcessor.FIELD_MAPPINGS, Collections.singletonMap("source", "dest"));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId, BytesReference.bytes(xContentBuilder), XContentType.JSON);
         }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java
index ccbebf6f6f738..e7c0eecc01744 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -42,16 +41,13 @@ public class LocalModelTests extends ESTestCase {
 
     public void testClassificationInfer() throws Exception {
         String modelId = "classification_model";
-        List<String> inputFields = Arrays.asList("field.foo.keyword", "field.bar", "categorical");
+        List<String> inputFields = Arrays.asList("field.foo", "field.bar", "categorical");
         TrainedModelDefinition definition = new TrainedModelDefinition.Builder()
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildClassification(false))
             .build();
 
-        Model model = new LocalModel(modelId,
-            definition,
-            new TrainedModelInput(inputFields),
-            Collections.singletonMap("field.foo", "field.foo.keyword"));
+        Model model = new LocalModel(modelId, definition, new TrainedModelInput(inputFields));
         Map<String, Object> fields = new HashMap<>() {{
             put("field.foo", 1.0);
             put("field.bar", 0.5);
@@ -72,10 +68,7 @@ public void testClassificationInfer() throws Exception {
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildClassification(true))
             .build();
-        model = new LocalModel(modelId,
-            definition,
-            new TrainedModelInput(inputFields),
-            Collections.singletonMap("field.foo", "field.foo.keyword"));
+        model = new LocalModel(modelId, definition, new TrainedModelInput(inputFields));
         result = getSingleValue(model, fields, new ClassificationConfig(0));
         assertThat(result.value(), equalTo(0.0));
         assertThat(result.valueAsString(), equalTo("not_to_be"));
@@ -97,14 +90,11 @@ public void testRegression() throws Exception {
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildRegression())
             .build();
-        Model model = new LocalModel("regression_model",
-            trainedModelDefinition,
-            new TrainedModelInput(inputFields),
-            Collections.singletonMap("bar", "bar.keyword"));
+        Model model = new LocalModel("regression_model", trainedModelDefinition, new TrainedModelInput(inputFields));
 
         Map<String, Object> fields = new HashMap<>() {{
-            put("foo", 1.0);
-            put("bar.keyword", 0.5);
+            put("field.foo", 1.0);
+            put("field.bar", 0.5);
             put("categorical", "dog");
         }};
 
@@ -124,7 +114,7 @@ public void testAllFieldsMissing() throws Exception {
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildRegression())
             .build();
-        Model model = new LocalModel("regression_model", trainedModelDefinition, new TrainedModelInput(inputFields), null);
+        Model model = new LocalModel("regression_model", trainedModelDefinition, new TrainedModelInput(inputFields));
 
         Map<String, Object> fields = new HashMap<>() {{
             put("something", 1.0);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
index 585bfd8545678..f5d3fc84b8eeb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
@@ -10,13 +10,16 @@
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
+import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
 import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
 import org.junit.Before;
 
+import java.util.Collection;
 import java.util.List;
 
 public class AnnotationIndexIT extends MlSingleNodeTestCase {
@@ -31,6 +34,11 @@ protected Settings nodeSettings()  {
         return newSettings.build();
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(LocalStateMachineLearning.class);
+    }
+
     @Before
     public void createComponents() throws Exception {
         waitForMlTemplates();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
index 23fe7704a5fd3..7f783f45bc774 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.ModelPlot;
-import org.elasticsearch.xpack.ilm.IndexLifecycle;
 import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
 import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor;
@@ -94,11 +93,7 @@ public class AutodetectResultProcessorIT extends MlSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(
-            LocalStateMachineLearning.class,
-            ReindexPlugin.class,
-            // ILM is required for .ml-state template index settings
-            IndexLifecycle.class);
+        return pluginList(LocalStateMachineLearning.class, ReindexPlugin.class);
     }
 
     @Before
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
index 23275958b3c10..899be32a217cb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
@@ -165,7 +165,7 @@ public void testJobAutoClose() throws Exception {
         indexRequest = new IndexRequest("data");
         indexRequest.source("time", 1407083600L);
         client().index(indexRequest).get();
-        refresh("*");
+        refresh("*", ".ml-*");
 
         Job.Builder job = createScheduledJob("job_id");
         PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
index 996ffec02af52..92e18a6e6430f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
@@ -6,17 +6,14 @@
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedConsumer;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -40,7 +37,6 @@
 import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
 import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
-import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
@@ -74,6 +70,8 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
@@ -84,9 +82,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
 import static org.elasticsearch.mock.orig.Mockito.doAnswer;
 import static org.elasticsearch.mock.orig.Mockito.doReturn;
 import static org.elasticsearch.mock.orig.Mockito.doThrow;
@@ -163,21 +158,10 @@ public void setup() throws Exception {
                 new HashSet<>(Arrays.asList(MachineLearning.MAX_OPEN_JOBS_PER_NODE,
                     ResultsPersisterService.PERSIST_RESULTS_MAX_RETRIES)));
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
-        MetaData metaData = MetaData.builder()
-            .indices(ImmutableOpenMap.<String, IndexMetaData>builder()
-                .fPut(
-                    AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-000001",
-                    IndexMetaData.builder(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-000001")
-                        .settings(
-                            Settings.builder()
-                                .put(SETTING_NUMBER_OF_SHARDS, 1)
-                                .put(SETTING_NUMBER_OF_REPLICAS, 0)
-                                .put(SETTING_VERSION_CREATED, Version.CURRENT)
-                                .build())
-                        .putAlias(AliasMetaData.builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).build())
-                        .build())
-                .build())
-            .build();
+        MetaData metaData = mock(MetaData.class);
+        SortedMap<String, AliasOrIndex> aliasOrIndexSortedMap = new TreeMap<>();
+        aliasOrIndexSortedMap.put(AnomalyDetectorsIndex.jobStateIndexWriteAlias(), mock(AliasOrIndex.Alias.class));
+        when(metaData.getAliasAndIndexLookup()).thenReturn(aliasOrIndexSortedMap);
         clusterState = mock(ClusterState.class);
         when(clusterState.getMetaData()).thenReturn(metaData);
         when(clusterState.metaData()).thenReturn(metaData);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
index 887cd17eb6b50..599dd2ab62204 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.test.MockHttpTransport;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
-import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
@@ -55,7 +54,6 @@
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
-import org.elasticsearch.xpack.ilm.IndexLifecycle;
 import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.junit.After;
@@ -94,18 +92,13 @@ protected Settings nodeSettings(int nodeOrdinal) {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
-        settings.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
         return settings.build();
     }
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(
-            LocalStateMachineLearning.class,
-            CommonAnalysisPlugin.class,
-            ReindexPlugin.class,
-            // ILM is required for .ml-state template index settings
-            IndexLifecycle.class);
+        return Arrays.asList(LocalStateMachineLearning.class, CommonAnalysisPlugin.class,
+                ReindexPlugin.class);
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
index 3439626887ac0..cbb50605f72e2 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
@@ -28,7 +28,6 @@
 import java.util.Objects;
 import java.util.StringJoiner;
 
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT;
@@ -214,7 +213,6 @@ protected Object unwrapMultiValue(Object values) {
     protected boolean isFromDocValuesOnly(DataType dataType) {
         return dataType == KEYWORD // because of ignore_above.
                     || dataType == DATETIME
-                    || dataType == CONSTANT_KEYWORD // because a non-existent value is considered the constant value itself
                     || dataType == SCALED_FLOAT; // because of scaling_factor
     }
     
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
index dfde94b5df76e..dbe751276be0d 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
@@ -6,6 +6,8 @@
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 
@@ -24,4 +26,15 @@ protected NodeInfo<Like> info() {
     protected Like replaceChild(Expression newLeft) {
         return new Like(source(), newLeft, pattern());
     }
+
+    @Override
+    public Boolean fold() {
+        Object val = field().fold();
+        return RegexOperation.match(val, pattern().asJavaRegex());
+    }
+
+    @Override
+    protected Processor makeProcessor() {
+        return new RegexProcessor(pattern().asJavaRegex());
+    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
index 096f7d8ae27e1..fd6663b4b5ac3 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
@@ -17,7 +17,7 @@
  *
  * To prevent conflicts with ES, the string and char must be validated to not contain '*'.
  */
-public class LikePattern implements StringPattern {
+public class LikePattern {
 
     private final String pattern;
     private final char escape;
@@ -43,7 +43,9 @@ public char escape() {
         return escape;
     }
 
-    @Override
+    /**
+     * Returns the pattern in (Java) regex format.
+     */
     public String asJavaRegex() {
         return regex;
     }
@@ -81,4 +83,4 @@ public boolean equals(Object obj) {
         return Objects.equals(pattern, other.pattern)
                 && escape == other.escape;
     }
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
index af40f4ffb1750..9ec70bc79d1d8 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
@@ -6,12 +6,14 @@
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 
-public class RLike extends RegexMatch<RLikePattern> {
+public class RLike extends RegexMatch<String> {
 
-    public RLike(Source source, Expression value, RLikePattern pattern) {
+    public RLike(Source source, Expression value, String pattern) {
         super(source, value, pattern);
     }
 
@@ -24,4 +26,15 @@ protected NodeInfo<RLike> info() {
     protected RLike replaceChild(Expression newChild) {
         return new RLike(source(), newChild, pattern());
     }
+    
+    @Override
+    public Boolean fold() {
+        Object val = field().fold();
+        return RegexOperation.match(val, pattern());
+    }
+
+    @Override
+    protected Processor makeProcessor() {
+        return new RegexProcessor(pattern());
+    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
deleted file mode 100644
index 69747f03e2150..0000000000000
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ql.expression.predicate.regex;
-
-public class RLikePattern implements StringPattern {
-
-    private final String regexpPattern;
-
-    public RLikePattern(String regexpPattern) {
-        this.regexpPattern = regexpPattern;
-    }
-
-    @Override
-    public String asJavaRegex() {
-        return regexpPattern;
-    }
-}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
index 565a75d489765..517c243992c24 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
@@ -10,19 +10,15 @@
 import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.Nullability;
 import org.elasticsearch.xpack.ql.expression.function.scalar.UnaryScalarFunction;
-import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
-import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.Objects;
 
-import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
-import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
 
-public abstract class RegexMatch<T extends StringPattern> extends UnaryScalarFunction {
+public abstract class RegexMatch<T> extends UnaryScalarFunction {
 
     private final T pattern;
     
@@ -58,30 +54,8 @@ public boolean foldable() {
         // right() is not directly foldable in any context but Like can fold it.
         return field().foldable();
     }
-
-    @Override
-    public Boolean fold() {
-        Object val = field().fold();
-        return RegexProcessor.RegexOperation.match(val, pattern().asJavaRegex());
-    }
-
-    @Override
-    protected Processor makeProcessor() {
-        return new RegexProcessor(pattern().asJavaRegex());
-    }
-
+    
     @Override
-    public ScriptTemplate asScript() {
-        ScriptTemplate fieldAsScript = asScript(field());
-        return new ScriptTemplate(
-                formatTemplate(format("{sql}.", "regex({},{})", fieldAsScript.template())),
-                paramsBuilder()
-                        .script(fieldAsScript.params())
-                        .variable(pattern.asJavaRegex())
-                        .build(),
-                dataType());
-    }
-
     public boolean equals(Object obj) {
         return super.equals(obj) && Objects.equals(((RegexMatch<?>) obj).pattern(), pattern());
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
deleted file mode 100644
index 6dac865b4ac5b..0000000000000
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ql.expression.predicate.regex;
-
-interface StringPattern {
-    /**
-     * Returns the pattern in (Java) regex format.
-     */
-    String asJavaRegex();
-}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
index 2b8cddfcf8b15..5d003dd80bf08 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
-import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypeRegistry;
 import org.elasticsearch.xpack.ql.type.DateEsField;
@@ -61,7 +60,6 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.action.ActionListener.wrap;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
@@ -300,13 +298,8 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
             StringBuilder errorMessage = new StringBuilder();
 
             boolean hasUnmapped = types.containsKey(UNMAPPED);
-            // a keyword field and a constant_keyword field with the same name in two different indices are considered "compatible"
-            // since a common use case of constant_keyword field involves two indices with a field having the same name: one being
-            // a keyword, the other being a constant_keyword
-            boolean hasCompatibleKeywords = types.containsKey(KEYWORD.esType()) && types.containsKey(CONSTANT_KEYWORD.esType());
-            int allowedTypesCount = (hasUnmapped ? 2 : 1) + (hasCompatibleKeywords ? 1 : 0);
 
-            if (types.size() > allowedTypesCount) {
+            if (types.size() > (hasUnmapped ? 2 : 1)) {
                 // build the error message
                 // and create a MultiTypeField
 
@@ -351,11 +344,6 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
                 }
             }
 
-            // if there are both a keyword and a constant_keyword type for this field, only keep the keyword as a common compatible type
-            if (hasCompatibleKeywords) {
-                types.remove(CONSTANT_KEYWORD.esType());
-            }
-
             // everything checks
             return null;
         });
@@ -447,9 +435,6 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa
         if (esType == DATETIME) {
             return new DateEsField(fieldName, props, isAggregateable);
         }
-        if (esType == CONSTANT_KEYWORD) {
-            return new ConstantKeywordEsField(fieldName);
-        }
         if (esType == UNSUPPORTED) {
             return new UnsupportedEsField(fieldName, typeName, null, props);
         }
@@ -516,14 +501,14 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
 
         for (Entry<String, Map<String, FieldCapabilities>> entry : sortedFields) {
             String fieldName = entry.getKey();
+            Map<String, FieldCapabilities> types = entry.getValue();
 
             // ignore size added by the mapper plugin
             if (FIELD_NAMES_BLACKLIST.contains(fieldName)) {
                 continue;
             }
 
-            Map<String, FieldCapabilities> types = new LinkedHashMap<>(entry.getValue());
-            // apply verification and possibly remove the "duplicate" CONSTANT_KEYWORD field type
+            // apply verification
             final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types);
 
             // filter meta fields and unmapped
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
index 042e886f5d0fb..ebbd84ef07c7c 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
@@ -111,7 +111,7 @@ public static Query doTranslate(RegexMatch e, TranslatorHandler handler) {
             }
 
             if (e instanceof RLike) {
-                String pattern = ((RLike) e).pattern().asJavaRegex();
+                String pattern = ((RLike) e).pattern();
                 q = new RegexQuery(e.source(), targetFieldName, pattern);
             }
 
@@ -351,4 +351,4 @@ private static Query boolQuery(Source source, Query left, Query right, boolean i
         }
         return new BoolQuery(source, isAnd, left, right);
     }
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
deleted file mode 100644
index 9983599dc4def..0000000000000
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.ql.type;
-
-import java.util.Collections;
-
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
-
-/**
- * SQL-related information about an index field with a constant_keyword type
- */
-public class ConstantKeywordEsField extends KeywordEsField {
-
-    public ConstantKeywordEsField(String name) {
-        super(name, CONSTANT_KEYWORD, Collections.emptyMap(), true, Short.MAX_VALUE, false, false);
-    }
-
-}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
index a325f9287355e..5400659c75ab4 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
@@ -21,7 +21,6 @@
 
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -60,12 +59,9 @@ public static DataType commonType(DataType left, DataType right) {
             return left;
         }
         if (isString(left) && isString(right)) {
-            if (left == TEXT || right == TEXT) {
+            if (left == TEXT) {
                 return TEXT;
             }
-            if (left == KEYWORD) {
-                return KEYWORD;
-            }
             return right;
         }
         if (left.isNumeric() && right.isNumeric()) {
@@ -124,7 +120,7 @@ public static Converter converterFor(DataType from, DataType to) {
             return DefaultConverter.TO_NULL;
         }
         // proper converters
-        if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
+        if (to == KEYWORD || to == TEXT) {
             return conversionToString(from);
         }
         if (to == LONG) {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
index bc04a3b0213a9..6253a5f89d7e2 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
@@ -18,34 +18,33 @@
 public final class DataTypes {
 
     // @formatter:off
-    public static final DataType UNSUPPORTED      = new DataType("UNSUPPORTED", null, 0,                 false, false, false);
+    public static final DataType UNSUPPORTED  = new DataType("UNSUPPORTED", null, 0,                 false, false, false);
 
-    public static final DataType NULL             = new DataType("null",              0,                 false, false, false);
+    public static final DataType NULL         = new DataType("null",              0,                 false, false, false);
 
-    public static final DataType BOOLEAN          = new DataType("boolean",           1,                 false, false, false);
+    public static final DataType BOOLEAN      = new DataType("boolean",           1,                 false, false, false);
     // integer numeric
-    public static final DataType BYTE             = new DataType("byte",              Byte.BYTES,        true, false, true);
-    public static final DataType SHORT            = new DataType("short",             Short.BYTES,       true, false, true);
-    public static final DataType INTEGER          = new DataType("integer",           Integer.BYTES,     true, false, true);
-    public static final DataType LONG             = new DataType("long",              Long.BYTES,        true, false, true);
+    public static final DataType BYTE         = new DataType("byte",              Byte.BYTES,        true, false, true);
+    public static final DataType SHORT        = new DataType("short",             Short.BYTES,       true, false, true);
+    public static final DataType INTEGER      = new DataType("integer",           Integer.BYTES,     true, false, true);
+    public static final DataType LONG         = new DataType("long",              Long.BYTES,        true, false, true);
     // decimal numeric
-    public static final DataType DOUBLE           = new DataType("double",            Double.BYTES,      false, true, true);
-    public static final DataType FLOAT            = new DataType("float",             Float.BYTES,       false, true, true);
-    public static final DataType HALF_FLOAT       = new DataType("half_float",        Float.BYTES,       false, true, true);
-    public static final DataType SCALED_FLOAT     = new DataType("scaled_float",      Long.BYTES,        false, true, true);
+    public static final DataType DOUBLE       = new DataType("double",            Double.BYTES,      false, true, true);
+    public static final DataType FLOAT        = new DataType("float",             Float.BYTES,       false, true, true);
+    public static final DataType HALF_FLOAT   = new DataType("half_float",        Float.BYTES,       false, true, true);
+    public static final DataType SCALED_FLOAT = new DataType("scaled_float",      Long.BYTES,        false, true, true);
     // string
-    public static final DataType KEYWORD          = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
-    public static final DataType TEXT             = new DataType("text",              Integer.MAX_VALUE, false, false, false);
-    public static final DataType CONSTANT_KEYWORD = new DataType("constant_keyword",  Integer.MAX_VALUE, false, false, true);
+    public static final DataType KEYWORD      = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
+    public static final DataType TEXT         = new DataType("text",              Integer.MAX_VALUE, false, false, false);
     // date
-    public static final DataType DATETIME         = new DataType("DATETIME", "date",  Long.BYTES,        false, false, true);
+    public static final DataType DATETIME     = new DataType("DATETIME", "date",  Long.BYTES,        false, false, true);
     // ip
     public static final DataType IP           = new DataType("ip",                45,                false, false, true);
     // binary
-    public static final DataType BINARY           = new DataType("binary",            Integer.MAX_VALUE, false, false, true);
+    public static final DataType BINARY       = new DataType("binary",            Integer.MAX_VALUE, false, false, true);
     // complex types
-    public static final DataType OBJECT           = new DataType("object",            0,                 false, false, false);
-    public static final DataType NESTED           = new DataType("nested",            0,                 false, false, false);
+    public static final DataType OBJECT       = new DataType("object",            0,                 false, false, false);
+    public static final DataType NESTED       = new DataType("nested",            0,                 false, false, false);
     //@formatter:on
     
     private static final Collection<DataType> TYPES = Arrays.asList(
@@ -62,7 +61,6 @@ public final class DataTypes {
             SCALED_FLOAT,
             KEYWORD,
             TEXT,
-            CONSTANT_KEYWORD,
             DATETIME,
             IP,
             BINARY,
@@ -134,7 +132,7 @@ public static boolean isUnsupported(DataType from) {
     }
 
     public static boolean isString(DataType t) {
-        return t == KEYWORD || t == TEXT || t == CONSTANT_KEYWORD;
+        return t == KEYWORD || t == TEXT;
     }
 
     public static boolean isPrimitive(DataType t) {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
index 65c679a5ad034..78d84946da4c9 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
@@ -29,12 +29,7 @@ public KeywordEsField(String name, Map<String, EsField> properties, boolean hasD
     
     public KeywordEsField(String name, Map<String, EsField> properties, boolean hasDocValues, int precision,
             boolean normalized, boolean isAlias) {
-        this(name, KEYWORD, properties, hasDocValues, precision, normalized, isAlias);
-    }
-
-    protected KeywordEsField(String name, DataType esDataType, Map<String, EsField> properties, boolean hasDocValues, int precision,
-            boolean normalized, boolean isAlias) {
-        super(name, esDataType, properties, hasDocValues, isAlias);
+        super(name, KEYWORD, properties, hasDocValues, isAlias);
         this.precision = precision;
         this.normalized = normalized;
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
index 777fa9baea8c6..e3e325be23f33 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
@@ -11,7 +11,6 @@
 import java.util.Map;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
 
@@ -45,7 +44,7 @@ public Exact getExactInfo() {
     private Tuple<EsField, String> findExact() {
         EsField field = null;
         for (EsField property : getProperties().values()) {
-            if ((property.getDataType() == KEYWORD || property.getDataType() == CONSTANT_KEYWORD) && property.getExactInfo().hasExact()) {
+            if (property.getDataType() == KEYWORD && property.getExactInfo().hasExact()) {
                 if (field != null) {
                     return new Tuple<>(null, "Multiple exact keyword candidates available for [" + getName() +
                         "]; specify which one to use");
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
index 136d6e8155451..44202b006fcf6 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
@@ -14,7 +14,6 @@
 import java.util.Map.Entry;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED;
@@ -90,8 +89,6 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
                 int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
                 boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
                 field = new KeywordEsField(name, properties, docValues, length, normalized);
-            } else if (esDataType == CONSTANT_KEYWORD) {
-                field = new ConstantKeywordEsField(name);
             } else if (esDataType == DATETIME) {
                 field = new DateEsField(name, properties, docValues);
             } else if (esDataType == UNSUPPORTED) {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
index b40edfc381fc5..abc9fee4d97db 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
@@ -20,7 +20,6 @@
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
@@ -54,7 +53,7 @@ static class ValueAndCompatibleTypes {
             new ValueAndCompatibleTypes(ESTestCase::randomLong, LONG, FLOAT, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomFloat, FLOAT, LONG, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomDouble, DOUBLE, LONG, FLOAT, BOOLEAN),
-            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD, CONSTANT_KEYWORD));
+            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD));
 
     public static Literal randomLiteral() {
         ValueAndCompatibleTypes gen = randomFrom(GENERATORS);
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
index 9121b42e7c9c8..b853443a0ea7c 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
@@ -49,13 +48,13 @@
 import java.util.List;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.xpack.ql.TestUtils.of;
 import static org.elasticsearch.xpack.ql.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.ql.expression.Literal.NULL;
 import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.ql.TestUtils.of;
 
 public class OptimizerRulesTests extends ESTestCase {
 
@@ -190,7 +189,7 @@ public void testConstantFoldingLikes() {
                 new ConstantFolding().rule(new Like(EMPTY, of("test_emp"), new LikePattern("test%", (char) 0)))
                         .canonical());
         assertEquals(TRUE,
-                new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
+                new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), "test.emp")).canonical());
     }
 
     public void testArithmeticFolding() {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
index 0010e99fdf510..f1188ed546e5a 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
@@ -17,7 +17,6 @@
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -362,7 +361,6 @@ public void testCommonType() {
         assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
         assertEquals(NULL, commonType(NULL, NULL));
         assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
-        assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
         assertEquals(LONG, commonType(TEXT, LONG));
         assertEquals(SHORT, commonType(SHORT, BYTE));
         assertEquals(FLOAT, commonType(BYTE, FLOAT));
@@ -372,11 +370,6 @@ public void testCommonType() {
         // strings
         assertEquals(TEXT, commonType(TEXT, KEYWORD));
         assertEquals(TEXT, commonType(KEYWORD, TEXT));
-        assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
-        assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
-        assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
-        assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
-        assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
     }
 
     public void testEsDataTypes() {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
index 7adbf9306d7b8..9958d00b02929 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
@@ -14,7 +14,6 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
@@ -136,10 +135,9 @@ public void testMultiField() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field.getDataType(), is(TEXT));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(3));
+        assertThat(fields.size(), is(2));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
-        assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
     }
 
     public void testMultiFieldTooManyOptions() {
@@ -150,10 +148,9 @@ public void testMultiFieldTooManyOptions() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field, instanceOf(TextEsField.class));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(3));
+        assertThat(fields.size(), is(2));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
-        assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
     }
 
     public void testNestedDoc() {
@@ -176,13 +173,6 @@ public void testIpField() {
         assertThat(dt.getDataType().typeName(), is("ip"));
     }
 
-    public void testConstantKeywordField() {
-        Map<String, EsField> mapping = loadMapping("mapping-constant-keyword.json");
-        assertThat(mapping.size(), is(1));
-        EsField dt = mapping.get("full_name");
-        assertThat(dt.getDataType().typeName(), is("constant_keyword"));
-    }
-
     public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json b/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
deleted file mode 100644
index 7f248dc26fba6..0000000000000
--- a/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "properties" : {
-        "full_name" : {
-            "type" : "constant_keyword",
-            "value" : "foo"
-        }
-    }
-}
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
index 8adc4c64e8476..9e293d4271333 100644
--- a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
+++ b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
@@ -9,10 +9,6 @@
                 "english" : {
                     "type" : "text",
                     "analyzer" : "english"
-                },
-                "constant" : {
-                    "type" : "constant_keyword",
-                    "value" : "some constant value"
                 }
             }
         }
diff --git a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java
deleted file mode 100644
index 7fcec7d6cae74..0000000000000
--- a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security;
-
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.rest.ESRestTestCase;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-
-public abstract class SecurityInBasicRestTestCase extends ESRestTestCase {
-    private RestHighLevelClient highLevelAdminClient;
-
-    @Override
-    protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("security_test_user", new SecureString("security-test-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    private RestHighLevelClient getHighLevelAdminClient() {
-        if (highLevelAdminClient == null) {
-            highLevelAdminClient = new RestHighLevelClient(
-                adminClient(),
-                ignore -> {
-                },
-                List.of()) {
-            };
-        }
-        return highLevelAdminClient;
-    }
-}
diff --git a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
index 98f92c750951c..837421f6000d5 100644
--- a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
@@ -11,6 +11,10 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
 import org.elasticsearch.xpack.security.authc.InternalRealms;
 
@@ -20,12 +24,29 @@
 import java.util.Base64;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SecurityWithBasicLicenseIT extends SecurityInBasicRestTestCase {
+public class SecurityWithBasicLicenseIT extends ESRestTestCase {
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("security_test_user", new SecureString("security-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
 
     public void testWithBasicLicense() throws Exception {
         checkLicenseType("basic");
diff --git a/x-pack/plugin/security/qa/security-trial/build.gradle b/x-pack/plugin/security/qa/security-trial/build.gradle
deleted file mode 100644
index e8e0b547c1828..0000000000000
--- a/x-pack/plugin/security/qa/security-trial/build.gradle
+++ /dev/null
@@ -1,28 +0,0 @@
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
-
-dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'default')
-  testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
-  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-}
-
-testClusters.integTest {
-  testDistribution = 'DEFAULT'
-  numberOfNodes = 2
-
-  setting 'xpack.ilm.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'xpack.security.enabled', 'true'
-  setting 'xpack.security.ssl.diagnose.trust', 'true'
-  setting 'xpack.security.http.ssl.enabled', 'false'
-  setting 'xpack.security.transport.ssl.enabled', 'false'
-  setting 'xpack.security.authc.token.enabled', 'true'
-  setting 'xpack.security.authc.api_key.enabled', 'true'
-
-  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
-  user username: "admin_user", password: "admin-password"
-  user username: "security_test_user", password: "security-test-password", role: "security_test_role"
-}
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
deleted file mode 100644
index 2fe99bc8ad48d..0000000000000
--- a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security;
-
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.client.security.CreateTokenRequest;
-import org.elasticsearch.client.security.CreateTokenResponse;
-import org.elasticsearch.client.security.DeleteRoleRequest;
-import org.elasticsearch.client.security.DeleteUserRequest;
-import org.elasticsearch.client.security.GetApiKeyRequest;
-import org.elasticsearch.client.security.GetApiKeyResponse;
-import org.elasticsearch.client.security.InvalidateApiKeyRequest;
-import org.elasticsearch.client.security.PutRoleRequest;
-import org.elasticsearch.client.security.PutUserRequest;
-import org.elasticsearch.client.security.RefreshPolicy;
-import org.elasticsearch.client.security.support.ApiKey;
-import org.elasticsearch.client.security.user.User;
-import org.elasticsearch.client.security.user.privileges.Role;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.hamcrest.Matchers;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-
-public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase {
-    private RestHighLevelClient highLevelAdminClient;
-
-    @Override
-    protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("security_test_user", new SecureString("security-test-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    protected void createUser(String username, SecureString password, List<String> roles) throws IOException {
-        final RestHighLevelClient client = getHighLevelAdminClient();
-        client.security().putUser(PutUserRequest.withPassword(new User(username, roles), password.getChars(), true,
-            RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT);
-    }
-
-    protected void createRole(String name, Collection<String> clusterPrivileges) throws IOException {
-        final RestHighLevelClient client = getHighLevelAdminClient();
-        final Role role = Role.builder().name(name).clusterPrivileges(clusterPrivileges).build();
-        client.security().putRole(new PutRoleRequest(role, null), RequestOptions.DEFAULT);
-    }
-
-    /**
-     * @return A tuple of (access-token, refresh-token)
-     */
-    protected Tuple<String, String> createOAuthToken(String username, SecureString password) throws IOException {
-        final RestHighLevelClient client = getHighLevelAdminClient();
-        final CreateTokenRequest request = CreateTokenRequest.passwordGrant(username, password.getChars());
-        final CreateTokenResponse response = client.security().createToken(request, RequestOptions.DEFAULT);
-        return new Tuple(response.getAccessToken(), response.getRefreshToken());
-    }
-
-    protected void deleteUser(String username) throws IOException {
-        final RestHighLevelClient client = getHighLevelAdminClient();
-        client.security().deleteUser(new DeleteUserRequest(username), RequestOptions.DEFAULT);
-    }
-
-    protected void deleteRole(String name) throws IOException {
-        final RestHighLevelClient client = getHighLevelAdminClient();
-        client.security().deleteRole(new DeleteRoleRequest(name), RequestOptions.DEFAULT);
-    }
-
-    protected void invalidateApiKeysForUser(String username) throws IOException {
-        final RestHighLevelClient client = getHighLevelAdminClient();
-        client.security().invalidateApiKey(InvalidateApiKeyRequest.usingUserName(username), RequestOptions.DEFAULT);
-    }
-
-    protected ApiKey getApiKey(String id) throws IOException {
-        final RestHighLevelClient client = getHighLevelAdminClient();
-        final GetApiKeyResponse response = client.security().getApiKey(GetApiKeyRequest.usingApiKeyId(id, false), RequestOptions.DEFAULT);
-        assertThat(response.getApiKeyInfos(), Matchers.iterableWithSize(1));
-        return response.getApiKeyInfos().get(0);
-    }
-
-    private RestHighLevelClient getHighLevelAdminClient() {
-        if (highLevelAdminClient == null) {
-            highLevelAdminClient = new RestHighLevelClient(
-                adminClient(),
-                ignore -> {
-                },
-                List.of()) {
-            };
-        }
-        return highLevelAdminClient;
-    }
-}
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
deleted file mode 100644
index daa0cfc303142..0000000000000
--- a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.apikey;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.security.support.ApiKey;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.XContentTestUtils;
-import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-import static org.hamcrest.Matchers.notNullValue;
-
-/**
- * Integration Rest Tests relating to API Keys.
- * Tested against a trial license
- */
-public class ApiKeyRestIT extends SecurityOnTrialLicenseRestTestCase {
-
-    private static final String SYSTEM_USER = "system_user";
-    private static final SecureString SYSTEM_USER_PASSWORD = new SecureString("sys-pass".toCharArray());
-    private static final String END_USER = "end_user";
-    private static final SecureString END_USER_PASSWORD = new SecureString("user-pass".toCharArray());
-
-    @Before
-    public void createUsers() throws IOException {
-        createUser(SYSTEM_USER, SYSTEM_USER_PASSWORD, List.of("system_role"));
-        createRole("system_role", Set.of("manage_api_key"));
-        createUser(END_USER, END_USER_PASSWORD, List.of("user_role"));
-        createRole("user_role", Set.of("monitor"));
-    }
-
-    @After
-    public void cleanUp() throws IOException {
-        deleteUser("system_user");
-        deleteUser("end_user");
-        deleteRole("system_role");
-        deleteRole("user_role");
-        invalidateApiKeysForUser(END_USER);
-    }
-
-    public void testGrantApiKeyForOtherUserWithPassword() throws IOException {
-        Request request = new Request("POST", "_security/api_key/grant");
-        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
-            UsernamePasswordToken.basicAuthHeaderValue(SYSTEM_USER, SYSTEM_USER_PASSWORD)));
-        final Map<String, Object> requestBody = Map.ofEntries(
-            Map.entry("grant_type", "password"),
-            Map.entry("username", END_USER),
-            Map.entry("password", END_USER_PASSWORD.toString()),
-            Map.entry("api_key", Map.of("name", "test_api_key_password"))
-        );
-        request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString());
-
-        final Response response = client().performRequest(request);
-        final Map<String, Object> responseBody = entityAsMap(response);
-
-        assertThat(responseBody.get("name"), equalTo("test_api_key_password"));
-        assertThat(responseBody.get("id"), notNullValue());
-        assertThat(responseBody.get("id"), instanceOf(String.class));
-
-        ApiKey apiKey = getApiKey((String) responseBody.get("id"));
-        assertThat(apiKey.getUsername(), equalTo(END_USER));
-    }
-
-    public void testGrantApiKeyForOtherUserWithAccessToken() throws IOException {
-        final Tuple<String, String> token = super.createOAuthToken(END_USER, END_USER_PASSWORD);
-        final String accessToken = token.v1();
-
-        final Request request = new Request("POST", "_security/api_key/grant");
-        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
-            UsernamePasswordToken.basicAuthHeaderValue(SYSTEM_USER, SYSTEM_USER_PASSWORD)));
-        final Map<String, Object> requestBody = Map.ofEntries(
-            Map.entry("grant_type", "access_token"),
-            Map.entry("access_token", accessToken),
-            Map.entry("api_key", Map.of("name", "test_api_key_token", "expiration", "2h"))
-        );
-        request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString());
-
-        final Instant before = Instant.now();
-        final Response response = client().performRequest(request);
-        final Instant after = Instant.now();
-        final Map<String, Object> responseBody = entityAsMap(response);
-
-        assertThat(responseBody.get("name"), equalTo("test_api_key_token"));
-        assertThat(responseBody.get("id"), notNullValue());
-        assertThat(responseBody.get("id"), instanceOf(String.class));
-
-        ApiKey apiKey = getApiKey((String) responseBody.get("id"));
-        assertThat(apiKey.getUsername(), equalTo(END_USER));
-
-        Instant minExpiry = before.plus(2, ChronoUnit.HOURS);
-        Instant maxExpiry = after.plus(2, ChronoUnit.HOURS);
-        assertThat(apiKey.getExpiration(), notNullValue());
-        assertThat(apiKey.getExpiration(), greaterThanOrEqualTo(minExpiry));
-        assertThat(apiKey.getExpiration(), lessThanOrEqualTo(maxExpiry));
-    }
-}
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml b/x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml
deleted file mode 100644
index 9b2171257fc61..0000000000000
--- a/x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml
+++ /dev/null
@@ -1,8 +0,0 @@
-# A basic role that is used to test security
-security_test_role:
-  cluster:
-    - monitor
-    - "cluster:admin/xpack/license/*"
-  indices:
-    - names: [ "index_allowed" ]
-      privileges: [ "read", "write", "create_index" ]
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 1f62f6c341f87..885de69564c2b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -81,7 +81,6 @@
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.DelegatePkiAuthenticationAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
-import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectAuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectLogoutAction;
@@ -143,7 +142,6 @@
 import org.elasticsearch.xpack.security.action.TransportCreateApiKeyAction;
 import org.elasticsearch.xpack.security.action.TransportDelegatePkiAuthenticationAction;
 import org.elasticsearch.xpack.security.action.TransportGetApiKeyAction;
-import org.elasticsearch.xpack.security.action.TransportGrantApiKeyAction;
 import org.elasticsearch.xpack.security.action.TransportInvalidateApiKeyAction;
 import org.elasticsearch.xpack.security.action.filter.SecurityActionFilter;
 import org.elasticsearch.xpack.security.action.oidc.TransportOpenIdConnectAuthenticateAction;
@@ -208,7 +206,6 @@
 import org.elasticsearch.xpack.security.rest.action.RestDelegatePkiAuthenticationAction;
 import org.elasticsearch.xpack.security.rest.action.apikey.RestCreateApiKeyAction;
 import org.elasticsearch.xpack.security.rest.action.apikey.RestGetApiKeyAction;
-import org.elasticsearch.xpack.security.rest.action.apikey.RestGrantApiKeyAction;
 import org.elasticsearch.xpack.security.rest.action.apikey.RestInvalidateApiKeyAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestInvalidateTokenAction;
@@ -756,7 +753,6 @@ public void onIndexModule(IndexModule module) {
                 new ActionHandler<>(PutPrivilegesAction.INSTANCE, TransportPutPrivilegesAction.class),
                 new ActionHandler<>(DeletePrivilegesAction.INSTANCE, TransportDeletePrivilegesAction.class),
                 new ActionHandler<>(CreateApiKeyAction.INSTANCE, TransportCreateApiKeyAction.class),
-                new ActionHandler<>(GrantApiKeyAction.INSTANCE, TransportGrantApiKeyAction.class),
                 new ActionHandler<>(InvalidateApiKeyAction.INSTANCE, TransportInvalidateApiKeyAction.class),
                 new ActionHandler<>(GetApiKeyAction.INSTANCE, TransportGetApiKeyAction.class),
                 new ActionHandler<>(DelegatePkiAuthenticationAction.INSTANCE, TransportDelegatePkiAuthenticationAction.class),
@@ -812,7 +808,6 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                 new RestPutPrivilegesAction(settings, getLicenseState()),
                 new RestDeletePrivilegesAction(settings, getLicenseState()),
                 new RestCreateApiKeyAction(settings, getLicenseState()),
-                new RestGrantApiKeyAction(settings, getLicenseState()),
                 new RestInvalidateApiKeyAction(settings, getLicenseState()),
                 new RestGetApiKeyAction(settings, getLicenseState()),
                 new RestDelegatePkiAuthenticationAction(settings, getLicenseState())
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
index 730affc7ea953..72a92516e59ed 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
@@ -6,10 +6,12 @@
 
 package org.elasticsearch.xpack.security.action;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
@@ -18,24 +20,32 @@
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
-import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 
+import java.util.Arrays;
+import java.util.HashSet;
+
 /**
  * Implementation of the action needed to create an API key
  */
 public final class TransportCreateApiKeyAction extends HandledTransportAction<CreateApiKeyRequest, CreateApiKeyResponse> {
 
-    private final ApiKeyGenerator generator;
+    private final ApiKeyService apiKeyService;
     private final SecurityContext securityContext;
+    private final CompositeRolesStore rolesStore;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportCreateApiKeyAction(TransportService transportService, ActionFilters actionFilters, ApiKeyService apiKeyService,
                                        SecurityContext context, CompositeRolesStore rolesStore, NamedXContentRegistry xContentRegistry) {
-        super(CreateApiKeyAction.NAME, transportService, actionFilters, CreateApiKeyRequest::new);
-        this.generator = new ApiKeyGenerator(apiKeyService, rolesStore, xContentRegistry);
+        super(CreateApiKeyAction.NAME, transportService, actionFilters, (Writeable.Reader<CreateApiKeyRequest>) CreateApiKeyRequest::new);
+        this.apiKeyService = apiKeyService;
         this.securityContext = context;
+        this.rolesStore = rolesStore;
+        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
@@ -44,7 +54,19 @@ protected void doExecute(Task task, CreateApiKeyRequest request, ActionListener<
         if (authentication == null) {
             listener.onFailure(new IllegalStateException("authentication is required"));
         } else {
-            generator.generateApiKey(authentication, request, listener);
+            rolesStore.getRoleDescriptors(new HashSet<>(Arrays.asList(authentication.getUser().roles())),
+                ActionListener.wrap(roleDescriptors -> {
+                        for (RoleDescriptor rd : roleDescriptors) {
+                            try {
+                                DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry);
+                            } catch (ElasticsearchException | IllegalArgumentException e) {
+                                listener.onFailure(e);
+                                return;
+                            }
+                        }
+                        apiKeyService.createApiKey(authentication, request, roleDescriptors, listener);
+                    },
+                    listener::onFailure));
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java
deleted file mode 100644
index d41754266329c..0000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.action;
-
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportMessage;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
-import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
-import org.elasticsearch.xpack.core.security.action.GrantApiKeyRequest;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.security.authc.ApiKeyService;
-import org.elasticsearch.xpack.security.authc.AuthenticationService;
-import org.elasticsearch.xpack.security.authc.TokenService;
-import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator;
-import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
-
-/**
- * Implementation of the action needed to create an API key on behalf of another user (using an OAuth style "grant")
- */
-public final class TransportGrantApiKeyAction extends HandledTransportAction<GrantApiKeyRequest, CreateApiKeyResponse> {
-
-    private final ThreadContext threadContext;
-    private final ApiKeyGenerator generator;
-    private final AuthenticationService authenticationService;
-    private final TokenService tokenService;
-
-    @Inject
-    public TransportGrantApiKeyAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool,
-                                      ApiKeyService apiKeyService, AuthenticationService authenticationService, TokenService tokenService,
-                                      CompositeRolesStore rolesStore, NamedXContentRegistry xContentRegistry) {
-        this(transportService, actionFilters, threadPool.getThreadContext(),
-            new ApiKeyGenerator(apiKeyService, rolesStore, xContentRegistry), authenticationService, tokenService
-        );
-    }
-
-    // Constructor for testing
-    TransportGrantApiKeyAction(TransportService transportService, ActionFilters actionFilters, ThreadContext threadContext,
-                                      ApiKeyGenerator generator, AuthenticationService authenticationService, TokenService tokenService) {
-        super(GrantApiKeyAction.NAME, transportService, actionFilters, GrantApiKeyRequest::new);
-        this.threadContext = threadContext;
-        this.generator = generator;
-        this.authenticationService = authenticationService;
-        this.tokenService = tokenService;
-    }
-
-    @Override
-    protected void doExecute(Task task, GrantApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            resolveAuthentication(request.getGrant(), request, ActionListener.wrap(
-                authentication -> generator.generateApiKey(authentication, request.getApiKeyRequest(), listener),
-                listener::onFailure
-            ));
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
-    }
-
-    private void resolveAuthentication(GrantApiKeyRequest.Grant grant, TransportMessage message, ActionListener<Authentication> listener) {
-        switch (grant.getType()) {
-            case GrantApiKeyRequest.PASSWORD_GRANT_TYPE:
-                final UsernamePasswordToken token = new UsernamePasswordToken(grant.getUsername(), grant.getPassword());
-                authenticationService.authenticate(super.actionName, message, token, listener);
-                return;
-            case GrantApiKeyRequest.ACCESS_TOKEN_GRANT_TYPE:
-                tokenService.authenticateToken(grant.getAccessToken(), listener);
-                return;
-            default:
-                listener.onFailure(new ElasticsearchSecurityException("the grant type [{}] is not supported", grant.getType()));
-                return;
-        }
-    }
-
-
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index 021984d073a23..5cd9c062914bc 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -574,7 +574,7 @@ private boolean isEnabled() {
         return enabled && licenseState.isApiKeyServiceAllowed();
     }
 
-    public void ensureEnabled() {
+    private void ensureEnabled() {
         if (licenseState.isApiKeyServiceAllowed() == false) {
             throw LicenseUtils.newComplianceException("api keys");
         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index c12b14477b8e7..24b40ba2922ae 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -396,35 +396,6 @@ void getAndValidateToken(ThreadContext ctx, ActionListener<UserToken> listener)
         }
     }
 
-    /**
-     * Decodes the provided token, and validates it (for format, expiry and invalidation).
-     * If valid, the token's {@link Authentication} (see {@link UserToken#getAuthentication()} is provided to the listener.
-     * If the token is invalid (expired etc), then {@link ActionListener#onFailure(Exception)} will be called.
-     * If tokens are not enabled, or the token does not exist, {@link ActionListener#onResponse} will be called with a
-     * {@code null} authentication object.
-     */
-    public void authenticateToken(SecureString tokenString, ActionListener<Authentication> listener) {
-        ensureEnabled();
-        decodeToken(tokenString.toString(), ActionListener.wrap(userToken -> {
-            if (userToken != null) {
-                checkIfTokenIsValid(userToken, ActionListener.wrap(
-                    token -> {
-                        if (token == null) {
-                            // Typically this means that the index is unavailable, so _probably_ the token is invalid but the only
-                            // this we can say for certain is that we couldn't validate it. The logs will be more explicit.
-                            listener.onFailure(new IllegalArgumentException("Cannot validate access token"));
-                        } else {
-                            listener.onResponse(token.getAuthentication());
-                        }
-                    },
-                    listener::onFailure
-                ));
-            } else {
-                listener.onFailure(new IllegalArgumentException("Cannot decode access token"));
-            }
-        }, listener::onFailure));
-    }
-
     /**
      * Reads the authentication and metadata from the given token.
      * This method does not validate whether the token is expired or not.
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java
deleted file mode 100644
index 1e1e4c0339c3d..0000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.authc.support;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
-import org.elasticsearch.xpack.security.authc.ApiKeyService;
-import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
-
-import java.util.Arrays;
-import java.util.HashSet;
-
-public class ApiKeyGenerator {
-
-    private final ApiKeyService apiKeyService;
-    private final CompositeRolesStore rolesStore;
-    private final NamedXContentRegistry xContentRegistry;
-
-    public ApiKeyGenerator(ApiKeyService apiKeyService, CompositeRolesStore rolesStore, NamedXContentRegistry xContentRegistry) {
-        this.apiKeyService = apiKeyService;
-        this.rolesStore = rolesStore;
-        this.xContentRegistry = xContentRegistry;
-    }
-
-    public void generateApiKey(Authentication authentication, CreateApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
-        if (authentication == null) {
-            listener.onFailure(new ElasticsearchSecurityException("no authentication available to generate API key"));
-            return;
-        }
-        apiKeyService.ensureEnabled();
-        rolesStore.getRoleDescriptors(new HashSet<>(Arrays.asList(authentication.getUser().roles())),
-            ActionListener.wrap(roleDescriptors -> {
-                    for (RoleDescriptor rd : roleDescriptors) {
-                        try {
-                            DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry);
-                        } catch (ElasticsearchException | IllegalArgumentException e) {
-                            listener.onFailure(e);
-                            return;
-                        }
-                    }
-                    apiKeyService.createApiKey(authentication, request, roleDescriptors, listener);
-                },
-                listener::onFailure));
-
-    }
-
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index 6113cf4dfd305..b61716462637e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -32,9 +32,6 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.transport.TransportActionProxy;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
@@ -242,18 +239,17 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
             // need to validate that the action is allowed and then move on
             authorizeIndexActionName(action, authorizationInfo, null, listener);
         } else if (request instanceof IndicesRequest == false && request instanceof IndicesAliasesRequest == false) {
+            // scroll is special
+            // some APIs are indices requests that are not actually associated with indices. For example,
+            // search scroll request, is categorized under the indices context, but doesn't hold indices names
+            // (in this case, the security check on the indices was done on the search request that initialized
+            // the scroll. Given that scroll is implemented using a context on the node holding the shard, we
+            // piggyback on it and enhance the context with the original authentication. This serves as our method
+            // to validate the scroll id only stays with the same user!
+            // note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any
+            // indices permission as it's categorized under cluster. This is why the scroll check is performed
+            // even before checking if the user has any indices permission.
             if (isScrollRelatedAction(action)) {
-                // scroll is special
-                // some APIs are indices requests that are not actually associated with indices. For example,
-                // search scroll request, is categorized under the indices context, but doesn't hold indices names
-                // (in this case, the security check on the indices was done on the search request that initialized
-                // the scroll. Given that scroll is implemented using a context on the node holding the shard, we
-                // piggyback on it and enhance the context with the original authentication. This serves as our method
-                // to validate the scroll id only stays with the same user!
-                // note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any
-                // indices permission as it's categorized under cluster. This is why the scroll check is performed
-                // even before checking if the user has any indices permission.
-
                 // if the action is a search scroll action, we first authorize that the user can execute the action for some
                 // index and if they cannot, we can fail the request early before we allow the execution of the action and in
                 // turn the shard actions
@@ -265,22 +261,11 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
                     // information such as the index and the incoming address of the request
                     listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
                 }
-            } else if (isAsyncSearchRelatedAction(action)) {
-                if (SubmitAsyncSearchAction.NAME.equals(action)) {
-                    // we check if the user has any indices permission when submitting an async-search request in order to be
-                    // able to fail the request early. Fine grained index-level permissions are handled by the search action
-                    // that is triggered internally by the submit API.
-                    authorizeIndexActionName(action, authorizationInfo, null, listener);
-                } else {
-                    // async-search actions other than submit have a custom security layer that checks if the current user is
-                    // the same as the user that submitted the original request so we can skip security here.
-                    listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
-                }
             } else {
-                assert false : "only scroll and async-search related requests are known indices api that don't " +
-                    "support retrieving the indices they relate to";
-                listener.onFailure(new IllegalStateException("only scroll and async-search related requests are known indices " +
-                    "api that don't support retrieving the indices they relate to"));
+                assert false :
+                    "only scroll related requests are known indices api that don't support retrieving the indices they relate to";
+                listener.onFailure(new IllegalStateException("only scroll related requests are known indices api that don't support " +
+                    "retrieving the indices they relate to"));
             }
         } else if (request instanceof IndicesRequest &&
             IndicesAndAliasesResolver.allowsRemoteIndices((IndicesRequest) request)) {
@@ -585,10 +570,4 @@ private static boolean isScrollRelatedAction(String action) {
             action.equals("indices:data/read/sql/close_cursor") ||
             action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
     }
-
-    private static boolean isAsyncSearchRelatedAction(String action) {
-        return action.equals(SubmitAsyncSearchAction.NAME) ||
-            action.equals(GetAsyncSearchAction.NAME) ||
-            action.equals(DeleteAsyncSearchAction.NAME);
-    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index d898003a50147..adfe0d4dd8e4c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -79,7 +78,7 @@ public void validateSearchContext(SearchContext searchContext, TransportRequest
      * (or lookup) realm. To work around this we compare the username and the originating realm type.
      */
     static void ensureAuthenticatedUserIsSame(Authentication original, Authentication current, AuditTrailService auditTrailService,
-                                              SearchContextId id, String action, TransportRequest request, String requestId,
+                                              long id, String action, TransportRequest request, String requestId,
                                               AuthorizationInfo authorizationInfo) {
         // this is really a best effort attempt since we cannot guarantee principal uniqueness
         // and realm names can change between nodes.
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java
deleted file mode 100644
index 0ce0c6588e810..0000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.rest.action.apikey;
-
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
-import org.elasticsearch.xpack.core.security.action.GrantApiKeyRequest;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.RestRequest.Method.PUT;
-
-/**
- * Rest action to create an API key on behalf of another user. Loosely mimics the API of
- * {@link org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction} combined with {@link RestCreateApiKeyAction}
- */
-public final class RestGrantApiKeyAction extends ApiKeyBaseRestHandler {
-
-    static final ObjectParser<GrantApiKeyRequest, Void> PARSER = new ObjectParser<>("grant_api_key_request", GrantApiKeyRequest::new);
-    static {
-        PARSER.declareString((req, str) -> req.getGrant().setType(str), new ParseField("grant_type"));
-        PARSER.declareString((req, str) -> req.getGrant().setUsername(str), new ParseField("username"));
-        PARSER.declareField((req, secStr) -> req.getGrant().setPassword(secStr), RestGrantApiKeyAction::getSecureString,
-            new ParseField("password"), ObjectParser.ValueType.STRING);
-        PARSER.declareField((req, secStr) -> req.getGrant().setAccessToken(secStr), RestGrantApiKeyAction::getSecureString,
-            new ParseField("access_token"), ObjectParser.ValueType.STRING);
-        PARSER.declareObject((req, api) -> req.setApiKeyRequest(api), (parser, ignore) -> CreateApiKeyRequestBuilder.parse(parser),
-            new ParseField("api_key"));
-    }
-
-    private static SecureString getSecureString(XContentParser parser) throws IOException {
-        return new SecureString(
-            Arrays.copyOfRange(parser.textCharacters(), parser.textOffset(), parser.textOffset() + parser.textLength()));
-    }
-
-    public RestGrantApiKeyAction(Settings settings, XPackLicenseState licenseState) {
-        super(settings, licenseState);
-    }
-
-    @Override
-    public List<Route> routes() {
-        return List.of(
-            new Route(POST, "/_security/api_key/grant"),
-            new Route(PUT, "/_security/api_key/grant"));
-    }
-
-    @Override
-    public String getName() {
-        return "xpack_security_grant_api_key";
-    }
-
-    @Override
-    protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        String refresh = request.param("refresh");
-        try (XContentParser parser = request.contentParser()) {
-            final GrantApiKeyRequest grantRequest = PARSER.parse(parser, null);
-            if (refresh != null) {
-                grantRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(refresh));
-            }
-            return channel -> client.execute(GrantApiKeyAction.INSTANCE, grantRequest,
-                ActionListener.delegateResponse(new RestToXContentListener<>(channel), (listener, ex) -> {
-                    RestStatus status = ExceptionsHelper.status(ex);
-                    if (status == RestStatus.UNAUTHORIZED) {
-                        listener.onFailure(
-                            new ElasticsearchSecurityException("Failed to authenticate api key grant", RestStatus.FORBIDDEN, ex));
-                    } else {
-                        listener.onFailure(ex);
-                    }
-                }));
-        }
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java
deleted file mode 100644
index 8d7faf15371f0..0000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.action;
-
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
-import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
-import org.elasticsearch.xpack.core.security.action.GrantApiKeyRequest;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authc.AuthenticationService;
-import org.elasticsearch.xpack.security.authc.TokenServiceMock;
-import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator;
-import org.elasticsearch.xpack.security.test.SecurityMocks;
-import org.junit.After;
-import org.junit.Before;
-
-import java.util.List;
-
-import static org.elasticsearch.test.TestMatchers.throwableWithMessage;
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verifyZeroInteractions;
-
-public class TransportGrantApiKeyActionTests extends ESTestCase {
-
-    private TransportGrantApiKeyAction action;
-    private ApiKeyGenerator apiKeyGenerator;
-    private AuthenticationService authenticationService;
-    private TokenServiceMock tokenServiceMock;
-    private ThreadPool threadPool;
-
-    @Before
-    public void setupMocks() throws Exception {
-        apiKeyGenerator = mock(ApiKeyGenerator.class);
-        authenticationService = mock(AuthenticationService.class);
-
-        threadPool = new TestThreadPool("TP-" + getTestName());
-        tokenServiceMock = SecurityMocks.tokenService(true, threadPool);
-        final ThreadContext threadContext = threadPool.getThreadContext();
-
-        action = new TransportGrantApiKeyAction(mock(TransportService.class), mock(ActionFilters.class), threadContext,
-            apiKeyGenerator, authenticationService, tokenServiceMock.tokenService);
-    }
-
-    @After
-    public void cleanup() {
-        threadPool.shutdown();
-    }
-
-    public void testGrantApiKeyWithUsernamePassword() throws Exception {
-        final String username = randomAlphaOfLengthBetween(4, 12);
-        final SecureString password = new SecureString(randomAlphaOfLengthBetween(8, 24).toCharArray());
-        final Authentication authentication = buildAuthentication(username);
-
-        final GrantApiKeyRequest request = mockRequest();
-        request.getGrant().setType("password");
-        request.getGrant().setUsername(username);
-        request.getGrant().setPassword(password);
-
-        final CreateApiKeyResponse response = mockResponse(request);
-
-        doAnswer(inv -> {
-            final Object[] args = inv.getArguments();
-            assertThat(args, arrayWithSize(4));
-
-            assertThat(args[0], equalTo(GrantApiKeyAction.NAME));
-            assertThat(args[1], sameInstance(request));
-            assertThat(args[2], instanceOf(UsernamePasswordToken.class));
-            UsernamePasswordToken token = (UsernamePasswordToken) args[2];
-            assertThat(token.principal(), equalTo(username));
-            assertThat(token.credentials(), equalTo(password));
-
-            ActionListener<Authentication> listener = (ActionListener<Authentication>) args[args.length - 1];
-            listener.onResponse(authentication);
-
-            return null;
-        }).when(authenticationService)
-            .authenticate(eq(GrantApiKeyAction.NAME), same(request), any(UsernamePasswordToken.class), any(ActionListener.class));
-
-        setupApiKeyGenerator(authentication, request, response);
-
-        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
-        action.doExecute(null, request, future);
-
-        assertThat(future.actionGet(), sameInstance(response));
-    }
-
-    public void testGrantApiKeyWithInvalidUsernamePassword() throws Exception {
-        final String username = randomAlphaOfLengthBetween(4, 12);
-        final SecureString password = new SecureString(randomAlphaOfLengthBetween(8, 24).toCharArray());
-        final Authentication authentication = buildAuthentication(username);
-
-        final GrantApiKeyRequest request = mockRequest();
-        request.getGrant().setType("password");
-        request.getGrant().setUsername(username);
-        request.getGrant().setPassword(password);
-
-        final CreateApiKeyResponse response = mockResponse(request);
-
-        doAnswer(inv -> {
-            final Object[] args = inv.getArguments();
-            assertThat(args, arrayWithSize(4));
-
-            assertThat(args[0], equalTo(GrantApiKeyAction.NAME));
-            assertThat(args[1], sameInstance(request));
-            assertThat(args[2], instanceOf(UsernamePasswordToken.class));
-            UsernamePasswordToken token = (UsernamePasswordToken) args[2];
-            assertThat(token.principal(), equalTo(username));
-            assertThat(token.credentials(), equalTo(password));
-
-            ActionListener<Authentication> listener = (ActionListener<Authentication>) args[args.length - 1];
-            listener.onFailure(new ElasticsearchSecurityException("authentication failed for testing"));
-
-            return null;
-        }).when(authenticationService)
-            .authenticate(eq(GrantApiKeyAction.NAME), same(request), any(UsernamePasswordToken.class), any(ActionListener.class));
-
-        setupApiKeyGenerator(authentication, request, response);
-
-        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
-        action.doExecute(null, request, future);
-
-        final ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, future::actionGet);
-        assertThat(exception, throwableWithMessage("authentication failed for testing"));
-
-        verifyZeroInteractions(apiKeyGenerator);
-    }
-
-    public void testGrantApiKeyWithAccessToken() throws Exception {
-        final String username = randomAlphaOfLengthBetween(4, 12);
-        final TokenServiceMock.MockToken token = tokenServiceMock.mockAccessToken();
-        final Authentication authentication = buildAuthentication(username);
-
-        final GrantApiKeyRequest request = mockRequest();
-        request.getGrant().setType("access_token");
-        request.getGrant().setAccessToken(token.encodedToken);
-
-        final CreateApiKeyResponse response = mockResponse(request);
-
-        tokenServiceMock.defineToken(token, authentication);
-        setupApiKeyGenerator(authentication, request, response);
-
-        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
-        action.doExecute(null, request, future);
-
-        assertThat(future.actionGet(), sameInstance(response));
-        verifyZeroInteractions(authenticationService);
-    }
-
-    public void testGrantApiKeyWithInvalidatedAccessToken() throws Exception {
-        final String username = randomAlphaOfLengthBetween(4, 12);
-        final TokenServiceMock.MockToken token = tokenServiceMock.mockAccessToken();
-        final Authentication authentication = buildAuthentication(username);
-
-        final GrantApiKeyRequest request = mockRequest();
-        request.getGrant().setType("access_token");
-        request.getGrant().setAccessToken(token.encodedToken);
-
-        final CreateApiKeyResponse response = mockResponse(request);
-
-        tokenServiceMock.defineToken(token, authentication, false);
-        setupApiKeyGenerator(authentication, request, response);
-
-        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
-        action.doExecute(null, request, future);
-
-        final ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, future::actionGet);
-        assertThat(exception, throwableWithMessage("token expired"));
-
-        verifyZeroInteractions(authenticationService);
-        verifyZeroInteractions(apiKeyGenerator);
-    }
-
-    private Authentication buildAuthentication(String username) {
-        return new Authentication(new User(username),
-            new Authentication.RealmRef("realm_name", "realm_type", "node_name"), null);
-    }
-
-    private CreateApiKeyResponse mockResponse(GrantApiKeyRequest request) {
-        return new CreateApiKeyResponse(request.getApiKeyRequest().getName(),
-            randomAlphaOfLength(12), new SecureString(randomAlphaOfLength(18).toCharArray()), null);
-    }
-
-    private GrantApiKeyRequest mockRequest() {
-        final String keyName = randomAlphaOfLengthBetween(6, 32);
-        final GrantApiKeyRequest request = new GrantApiKeyRequest();
-        request.setApiKeyRequest(new CreateApiKeyRequest(keyName, List.of(), null));
-        return request;
-    }
-
-    private void setupApiKeyGenerator(Authentication authentication, GrantApiKeyRequest request, CreateApiKeyResponse response) {
-        doAnswer(inv -> {
-            final Object[] args = inv.getArguments();
-            assertThat(args, arrayWithSize(3));
-
-            assertThat(args[0], equalTo(authentication));
-            assertThat(args[1], sameInstance(request.getApiKeyRequest()));
-
-            ActionListener<CreateApiKeyResponse> listener = (ActionListener<CreateApiKeyResponse>) args[args.length - 1];
-            listener.onResponse(response);
-
-            return null;
-        }).when(apiKeyGenerator).generateApiKey(any(Authentication.class), any(CreateApiKeyRequest.class), any(ActionListener.class));
-    }
-
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
deleted file mode 100644
index 76e321f69c439..0000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.authc;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.XContentTestUtils;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
-import org.elasticsearch.xpack.security.test.SecurityMocks;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Because {@link TokenService} is {@code final}, we can't mock it.
- * Instead, we use this class to control the client that underlies the token service and trigger certain conditions
- */
-public class TokenServiceMock {
-    public final TokenService tokenService;
-    public final Client client;
-
-    public final class MockToken {
-        public final String baseToken;
-        public final SecureString encodedToken;
-        public final String hashedToken;
-
-        public MockToken(String baseToken, SecureString encodedToken, String hashedToken) {
-            this.baseToken = baseToken;
-            this.encodedToken = encodedToken;
-            this.hashedToken = hashedToken;
-        }
-    }
-
-    public TokenServiceMock(TokenService tokenService, Client client) {
-        this.tokenService = tokenService;
-        this.client = client;
-    }
-
-    public MockToken mockAccessToken() throws Exception {
-        final String uuid = UUIDs.randomBase64UUID();
-        final SecureString encoded = new SecureString(tokenService.prependVersionAndEncodeAccessToken(Version.CURRENT, uuid).toCharArray());
-        final String hashedToken = TokenService.hashTokenString(uuid);
-        return new MockToken(uuid, encoded, hashedToken);
-    }
-
-    public void defineToken(MockToken token, Authentication authentication) throws IOException {
-        defineToken(token, authentication, true);
-    }
-
-    public void defineToken(MockToken token, Authentication authentication, boolean valid) throws IOException {
-        Instant expiration = Instant.now().plusSeconds(TimeUnit.MINUTES.toSeconds(20));
-        final UserToken userToken = new UserToken(token.hashedToken, Version.CURRENT, authentication, expiration, Map.of());
-        final Map<String, Object> document = new HashMap<>();
-        document.put("access_token", Map.of("user_token", userToken, "invalidated", valid == false));
-
-        SecurityMocks.mockGetRequest(client, RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, "token_" + token.hashedToken,
-            XContentTestUtils.convertToXContent(document, XContentType.JSON));
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java
deleted file mode 100644
index 6d9a92d3a625d..0000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.authc.support;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authc.ApiKeyService;
-import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
-
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anySetOf;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-
-public class ApiKeyGeneratorTests extends ESTestCase {
-
-    public void testGenerateApiKeySuccessfully() {
-        final ApiKeyService apiKeyService = mock(ApiKeyService.class);
-        final CompositeRolesStore rolesStore = mock(CompositeRolesStore.class);
-        final ApiKeyGenerator generator = new ApiKeyGenerator(apiKeyService, rolesStore, NamedXContentRegistry.EMPTY);
-        final Set<String> userRoleNames = Sets.newHashSet(randomArray(1, 4, String[]::new, () -> randomAlphaOfLengthBetween(3, 12)));
-        final Authentication authentication = new Authentication(
-            new User("test", userRoleNames.toArray(String[]::new)),
-            new Authentication.RealmRef("realm-name", "realm-type", "node-name"),
-            null);
-        final CreateApiKeyRequest request = new CreateApiKeyRequest("name", null, null);
-
-        final Set<RoleDescriptor> roleDescriptors = randomSubsetOf(userRoleNames).stream()
-            .map(name -> new RoleDescriptor(name, generateRandomStringArray(3, 6, false), null, null))
-            .collect(Collectors.toUnmodifiableSet());
-
-        doAnswer(inv -> {
-            final Object[] args = inv.getArguments();
-            assertThat(args, arrayWithSize(2));
-
-            Set<String> roleNames = (Set<String>) args[0];
-            assertThat(roleNames, equalTo(userRoleNames));
-
-            ActionListener<Set<RoleDescriptor>> listener = (ActionListener<Set<RoleDescriptor>>) args[args.length - 1];
-            listener.onResponse(roleDescriptors);
-            return null;
-        }).when(rolesStore).getRoleDescriptors(anySetOf(String.class), any(ActionListener.class));
-
-        CreateApiKeyResponse response = new CreateApiKeyResponse(
-            "name", randomAlphaOfLength(18), new SecureString(randomAlphaOfLength(24).toCharArray()), null);
-        doAnswer(inv -> {
-            final Object[] args = inv.getArguments();
-            assertThat(args, arrayWithSize(4));
-
-            assertThat(args[0], sameInstance(authentication));
-            assertThat(args[1], sameInstance(request));
-            assertThat(args[2], sameInstance(roleDescriptors));
-
-            ActionListener<CreateApiKeyResponse> listener = (ActionListener<CreateApiKeyResponse>) args[args.length - 1];
-            listener.onResponse(response);
-
-            return null;
-        }).when(apiKeyService).createApiKey(same(authentication), same(request), anySetOf(RoleDescriptor.class), any(ActionListener.class));
-
-        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
-        generator.generateApiKey(authentication, request, future);
-
-        assertThat(future.actionGet(), sameInstance(response));
-    }
-
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index c766ab211c173..aa3b612e513ba 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.security.authz;
 
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -16,7 +15,6 @@
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.transport.TransportRequest;
@@ -129,7 +127,7 @@ public void testValidateSearchContext() throws Exception {
             final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
             SearchContextMissingException expected =
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
-            assertEquals(testSearchContext.id(), expected.contextId());
+            assertEquals(testSearchContext.id(), expected.id());
             verify(licenseState, times(3)).isAuthAllowed();
             verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                 authzInfoRoles(authentication.getUser().roles()));
@@ -165,7 +163,7 @@ public void testValidateSearchContext() throws Exception {
             final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
             SearchContextMissingException expected =
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
-            assertEquals(testSearchContext.id(), expected.contextId());
+            assertEquals(testSearchContext.id(), expected.id());
             verify(licenseState, times(5)).isAuthAllowed();
             verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                 authzInfoRoles(authentication.getUser().roles()));
@@ -176,13 +174,13 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication original = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
         Authentication current =
                 randomBoolean() ? original : new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
-        SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomLong());
+        long id = randomLong();
         final String action = randomAlphaOfLength(4);
         TransportRequest request = Empty.INSTANCE;
         AuditTrailService auditTrail = mock(AuditTrailService.class);
 
         final String auditId = randomAlphaOfLengthBetween(8, 20);
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, contextId, action, request, auditId,
+        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -190,7 +188,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         User user = new User(new User("test", "role"), new User("authenticated", "runas"));
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, contextId, action, request, auditId,
+        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -198,7 +196,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
         Authentication runAs = current;
-        ensureAuthenticatedUserIsSame(runAs, current, auditTrail, contextId, action, request, auditId,
+        ensureAuthenticatedUserIsSame(runAs, current, auditTrail, id, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -206,9 +204,9 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication differentRealmType =
                 new Authentication(new User("test", "role"), new RealmRef("realm", randomAlphaOfLength(5), "node"), null);
         SearchContextMissingException e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, differentRealmType, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, differentRealmType, auditTrail, id, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(contextId, e.contextId());
+        assertEquals(id, e.id());
         verify(auditTrail).accessDenied(eq(auditId), eq(differentRealmType), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
 
@@ -216,9 +214,9 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication differentUser =
                 new Authentication(new User("test2", "role"), new RealmRef("realm", "realm", "node"), null);
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, differentUser, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, differentUser, auditTrail, id, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(contextId, e.contextId());
+        assertEquals(id, e.id());
         verify(auditTrail).accessDenied(eq(auditId), eq(differentUser), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
 
@@ -226,18 +224,18 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")),
                 new RealmRef("realm", "file", "node1"), new RealmRef("realm", "file", "node1"));
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, diffRunAs, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, diffRunAs, auditTrail, id, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(contextId, e.contextId());
+        assertEquals(id, e.id());
         verify(auditTrail).accessDenied(eq(auditId), eq(diffRunAs), eq(action), eq(request), authzInfoRoles(original.getUser().roles()));
 
         // run as different looked up by type
         Authentication runAsDiffType = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node"));
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(runAs, runAsDiffType, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(runAs, runAsDiffType, auditTrail, id, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(contextId, e.contextId());
+        assertEquals(id, e.id());
         verify(auditTrail).accessDenied(eq(auditId), eq(runAsDiffType), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java
index c9bb7879147b6..0270a07d9f97c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java
@@ -17,29 +17,17 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.security.SecurityContext;
-import org.elasticsearch.xpack.security.authc.TokenService;
-import org.elasticsearch.xpack.security.authc.TokenServiceMock;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.Assert;
 
-import java.security.GeneralSecurityException;
-import java.time.Clock;
-import java.time.Instant;
 import java.util.function.Consumer;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
-import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_TOKENS_ALIAS;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -141,20 +129,4 @@ public static void mockIndexRequest(Client client, String indexAliasName, Consum
             return null;
         }).when(client).execute(eq(IndexAction.INSTANCE), any(IndexRequest.class), any(ActionListener.class));
     }
-
-    public static TokenServiceMock tokenService(boolean enabled, ThreadPool threadPool) throws GeneralSecurityException {
-        final Settings settings = Settings.builder().put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), enabled).build();
-        final Instant now = Instant.now();
-        final Clock clock = Clock.fixed(now, ESTestCase.randomZone());
-        final Client client = mock(Client.class);
-        when(client.threadPool()).thenReturn(threadPool);
-        final XPackLicenseState licenseState = mock(XPackLicenseState.class);
-        when(licenseState.isTokenServiceAllowed()).thenReturn(true);
-        final ClusterService clusterService = mock(ClusterService.class);
-
-        final SecurityContext securityContext = new SecurityContext(settings, threadPool.getThreadContext());
-        final TokenService service = new TokenService(settings, clock, client, licenseState, securityContext,
-            mockSecurityIndexManager(SECURITY_MAIN_ALIAS), mockSecurityIndexManager(SECURITY_TOKENS_ALIAS), clusterService);
-        return new TokenServiceMock(service, client);
-    }
 }
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java
index 365268396c10c..6c5725c0b7528 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java
@@ -5,10 +5,9 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ConnectionConfiguration;
+import org.elasticsearch.xpack.sql.client.Version;
 
-import javax.sql.DataSource;
 import java.io.PrintWriter;
 import java.sql.Connection;
 import java.sql.SQLException;
@@ -17,6 +16,8 @@
 import java.util.Properties;
 import java.util.logging.Logger;
 
+import javax.sql.DataSource;
+
 /**
  * Factory for connections to Elasticsearch SQL.
  */
@@ -24,7 +25,7 @@ public class EsDataSource implements DataSource, Wrapper {
 
     static {
         // invoke Version to perform classpath/jar sanity checks
-        ClientVersion.CURRENT.toString();
+        Version.CURRENT.toString();
     }
 
     private String url;
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java
index 1599fa2ad02af..238a8fea163cf 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
-import org.elasticsearch.xpack.sql.client.ClientVersion;
+import org.elasticsearch.xpack.sql.client.Version;
 
 import java.io.PrintWriter;
 import java.sql.Connection;
@@ -23,7 +23,7 @@ public class EsDriver implements Driver {
 
     static {
         // invoke Version to perform classpath/jar sanity checks
-        ClientVersion.CURRENT.toString();
+        Version.CURRENT.toString();
 
         try {
             register();
@@ -96,12 +96,12 @@ public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws
 
     @Override
     public int getMajorVersion() {
-        return ClientVersion.CURRENT.major;
+        return Version.CURRENT.major;
     }
 
     @Override
     public int getMinorVersion() {
-        return ClientVersion.CURRENT.minor;
+        return Version.CURRENT.minor;
     }
 
     @Override
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
index 9eae74d18019a..0b96204fd5d7e 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
@@ -25,7 +25,6 @@ public enum EsType implements SQLType {
     SCALED_FLOAT(             Types.FLOAT),
     KEYWORD(                  Types.VARCHAR),
     TEXT(                     Types.VARCHAR),
-    CONSTANT_KEYWORD(         Types.VARCHAR),
     OBJECT(                   Types.STRUCT),
     NESTED(                   Types.STRUCT),
     BINARY(                   Types.VARBINARY),
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java
index 4ca094b6588df..aff896d1c21f3 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java
@@ -5,23 +5,28 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
-
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
-
 /**
  * General information about the server.
  */
 class InfoResponse {
     final String cluster;
-    final SqlVersion version;
+    final int majorVersion;
+    final int minorVersion;
+    final int revisionVersion;
 
-    InfoResponse(String clusterName, SqlVersion version) {
+    InfoResponse(String clusterName, byte versionMajor, byte versionMinor, byte revisionVersion) {
         this.cluster = clusterName;
-        this.version = version;
+        this.majorVersion = versionMajor;
+        this.minorVersion = versionMinor;
+        this.revisionVersion = revisionVersion;
     }
 
     @Override
     public String toString() {
-        return cluster + "[" + version.toString() + "]";
+        return cluster + "[" + versionString() + "]";
+    }
+    
+    public String versionString() {
+        return majorVersion + "." + minorVersion + "." + revisionVersion;
     }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
index 8fd4c6119b434..c9480dbcb1c2b 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ConnectionConfiguration;
 import org.elasticsearch.xpack.sql.client.StringUtils;
+import org.elasticsearch.xpack.sql.client.Version;
 
 import java.net.URI;
 import java.sql.DriverPropertyInfo;
@@ -70,7 +70,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
         // typically this should have already happened but in case the
         // EsDriver/EsDataSource are not used and the impl. classes used directly
         // this covers that case
-        ClientVersion.CURRENT.toString();
+        Version.CURRENT.toString();
     }
 
     // immutable properties
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java
index e4ab36996804f..09096fbe405a0 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java
@@ -426,10 +426,10 @@ String getUserName() {
     // in fact, this information is cached by the underlying client
     // once retrieved
     int esInfoMajorVersion() throws SQLException {
-        return client.serverInfo().version.major;
+        return client.serverInfo().majorVersion;
     }
 
     int esInfoMinorVersion() throws SQLException {
-        return client.serverInfo().version.minor;
+        return client.serverInfo().minorVersion;
     }
 }
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java
index 77563b60041db..8fd4417f4a9b0 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ObjectUtils;
+import org.elasticsearch.xpack.sql.client.Version;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -94,7 +94,7 @@ public String getDatabaseProductName() throws SQLException {
 
     @Override
     public String getDatabaseProductVersion() throws SQLException {
-        return ClientVersion.CURRENT.toString();
+        return Version.CURRENT.toString();
     }
 
     @Override
@@ -104,17 +104,17 @@ public String getDriverName() throws SQLException {
 
     @Override
     public String getDriverVersion() throws SQLException {
-        return ClientVersion.CURRENT.major + "." + ClientVersion.CURRENT.minor;
+        return Version.CURRENT.major + "." + Version.CURRENT.minor;
     }
 
     @Override
     public int getDriverMajorVersion() {
-        return ClientVersion.CURRENT.major;
+        return Version.CURRENT.major;
     }
 
     @Override
     public int getDriverMinorVersion() {
-        return ClientVersion.CURRENT.minor;
+        return Version.CURRENT.minor;
     }
 
     @Override
@@ -1111,12 +1111,12 @@ public int getDatabaseMinorVersion() throws SQLException {
 
     @Override
     public int getJDBCMajorVersion() throws SQLException {
-        return ClientVersion.jdbcMajorVersion();
+        return Version.jdbcMajorVersion();
     }
 
     @Override
     public int getJDBCMinorVersion() throws SQLException {
-        return ClientVersion.jdbcMinorVersion();
+        return Version.jdbcMinorVersion();
     }
 
     @Override
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
index 684f32981dabc..925187db405aa 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
@@ -7,8 +7,8 @@
 
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.HttpClient;
+import org.elasticsearch.xpack.sql.client.Version;
 import org.elasticsearch.xpack.sql.proto.ColumnInfo;
 import org.elasticsearch.xpack.sql.proto.MainResponse;
 import org.elasticsearch.xpack.sql.proto.Mode;
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.sql.proto.SqlQueryRequest;
 import org.elasticsearch.xpack.sql.proto.SqlQueryResponse;
 import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue;
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -63,7 +62,7 @@ Cursor query(String sql, List<SqlTypedParamValue> params, RequestMeta meta) thro
                 null,
                 Boolean.FALSE,
                 null,
-                new RequestInfo(Mode.JDBC, ClientVersion.CURRENT),
+                new RequestInfo(Mode.JDBC),
                 conCfg.fieldMultiValueLeniency(),
                 conCfg.indexIncludeFrozen(),
                 conCfg.binaryCommunication());
@@ -95,15 +94,16 @@ InfoResponse serverInfo() throws SQLException {
 
     private InfoResponse fetchServerInfo() throws SQLException {
         MainResponse mainResponse = httpClient.serverInfo();
-        SqlVersion version = SqlVersion.fromString(mainResponse.getVersion());
-        return new InfoResponse(mainResponse.getClusterName(), version);
+        Version version = Version.fromString(mainResponse.getVersion());
+        return new InfoResponse(mainResponse.getClusterName(), version.major, version.minor, version.revision);
     }
-
+    
     private void checkServerVersion() throws SQLException {
-        if (ClientVersion.isServerCompatible(serverInfo.version) == false) {
+        if (serverInfo.majorVersion != Version.CURRENT.major
+                || serverInfo.minorVersion != Version.CURRENT.minor
+                || serverInfo.revisionVersion != Version.CURRENT.revision) {
             throw new SQLException("This version of the JDBC driver is only compatible with Elasticsearch version " +
-                ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server version " +
-                serverInfo.version.toString());
+                    Version.CURRENT.toString() + ", attempting to connect to a server version " + serverInfo.versionString());
         }
     }
 
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
index 888fb8632d217..bf72b4454e40e 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
@@ -210,7 +210,6 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
             case BOOLEAN:
             case TEXT:
             case KEYWORD:
-            case CONSTANT_KEYWORD:
                 return v; // These types are already represented correctly in JSON
             case BYTE:
                 return ((Number) v).byteValue(); // Parser might return it as integer or long - need to update to the correct type
@@ -324,7 +323,6 @@ private static Boolean asBoolean(Object val, EsType columnType, String typeStrin
                 return Boolean.valueOf(Integer.signum(((Number) val).intValue()) != 0);
             case KEYWORD:
             case TEXT:
-            case CONSTANT_KEYWORD:
                 return Boolean.valueOf((String) val);
             default:
                 return failConversion(val, columnType, typeString, Boolean.class);
@@ -347,7 +345,6 @@ private static Byte asByte(Object val, EsType columnType, String typeString) thr
                 return safeToByte(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
-            case CONSTANT_KEYWORD:
                 try {
                     return Byte.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -375,7 +372,6 @@ private static Short asShort(Object val, EsType columnType, String typeString) t
                 return safeToShort(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
-            case CONSTANT_KEYWORD:
                 try {
                     return Short.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -402,7 +398,6 @@ private static Integer asInteger(Object val, EsType columnType, String typeStrin
                 return safeToInt(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
-            case CONSTANT_KEYWORD:
                 try {
                     return Integer.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -434,7 +429,6 @@ private static Long asLong(Object val, EsType columnType, String typeString) thr
             //    return ((Number) val).longValue();
             case KEYWORD:
             case TEXT:
-            case CONSTANT_KEYWORD:
                 try {
                     return Long.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -462,7 +456,6 @@ private static Float asFloat(Object val, EsType columnType, String typeString) t
                 return Float.valueOf(((Number) val).floatValue());
             case KEYWORD:
             case TEXT:
-            case CONSTANT_KEYWORD:
                 try {
                     return Float.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -489,7 +482,6 @@ private static Double asDouble(Object val, EsType columnType, String typeString)
                 return Double.valueOf(((Number) val).doubleValue());
             case KEYWORD:
             case TEXT:
-            case CONSTANT_KEYWORD:
                 try {
                     return Double.valueOf((String) val);
                 } catch (NumberFormatException e) {
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
index 0fc929c900bfe..511cf92be5665 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
@@ -76,7 +76,6 @@ private TypeUtils() {}
         types.put(EsType.SCALED_FLOAT, Double.class);
         types.put(EsType.KEYWORD, String.class);
         types.put(EsType.TEXT, String.class);
-        types.put(EsType.CONSTANT_KEYWORD, String.class);
         types.put(EsType.BINARY, byte[].class);
         types.put(EsType.DATETIME, Timestamp.class);
         types.put(EsType.IP, String.class);
@@ -158,7 +157,7 @@ static EsType of(String name) throws SQLException {
     }
 
     static boolean isString(EsType dataType) {
-        return dataType == EsType.KEYWORD || dataType == EsType.TEXT || dataType == EsType.CONSTANT_KEYWORD;
+        return dataType == EsType.KEYWORD || dataType == EsType.TEXT;
     }
 
     static EsType of(Class<? extends Object> clazz) throws SQLException {
diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java
index 4f6cdd6b51225..16df417855d12 100644
--- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java
+++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.http.MockResponse;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 
 import java.io.IOException;
 import java.sql.SQLException;
@@ -32,9 +31,9 @@ public void testExceptionThrownOnIncompatibleVersions() throws IOException, SQLE
         
         String url = JdbcConfiguration.URL_PREFIX + webServerAddress();
         SQLException ex = expectThrows(SQLException.class, () -> new JdbcHttpClient(JdbcConfiguration.create(url, null, 0)));
-        assertEquals("This version of the JDBC driver is only compatible with Elasticsearch version " +
-            ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server " +
-            "version " + version.toString(), ex.getMessage());
+        assertEquals("This version of the JDBC driver is only compatible with Elasticsearch version "
+                + org.elasticsearch.xpack.sql.client.Version.CURRENT.toString()
+                + ", attempting to connect to a server version " + version.toString(), ex.getMessage());
     }
     
     public void testNoExceptionThrownForCompatibleVersions() throws IOException {
diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java
index 8a47025a17135..0a2d3206468d8 100644
--- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java
+++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java
@@ -6,14 +6,15 @@
 package org.elasticsearch.xpack.sql.jdbc;
 
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
+import org.elasticsearch.xpack.sql.client.Version;
 
 public class VersionTests extends ESTestCase {
     public void testVersionIsCurrent() {
         /* This test will only work properly in gradle because in gradle we run the tests
          * using the jar. */
-        assertEquals(org.elasticsearch.Version.CURRENT.major, ClientVersion.CURRENT.major);
-        assertEquals(org.elasticsearch.Version.CURRENT.minor, ClientVersion.CURRENT.minor);
-        assertEquals(org.elasticsearch.Version.CURRENT.revision, ClientVersion.CURRENT.revision);
+        assertNotNull(Version.CURRENT.hash);
+        assertEquals(org.elasticsearch.Version.CURRENT.major, Version.CURRENT.major);
+        assertEquals(org.elasticsearch.Version.CURRENT.minor, Version.CURRENT.minor);
+        assertEquals(org.elasticsearch.Version.CURRENT.revision, Version.CURRENT.revision);
     }
 }
diff --git a/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
index 91f1a044c59f5..01ef407d4a027 100644
--- a/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
@@ -26,7 +26,6 @@
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.toMap;
-import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
 
@@ -111,7 +110,7 @@ private void assertCount(RestClient client, int count) throws IOException {
         expected.put("rows", singletonList(singletonList(count)));
 
         Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
-        request.setJsonEntity("{\"query\": \"SELECT COUNT(*) FROM test\"" + mode(mode) + version(mode) + "}");
+        request.setJsonEntity("{\"query\": \"SELECT COUNT(*) FROM test\"" + mode(mode) + "}");
         Map<String, Object> actual = toMap(client.performRequest(request), mode);
 
         if (false == expected.equals(actual)) {
diff --git a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
index 28c3b8fc018ac..f29f8ae423cf3 100644
--- a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
@@ -32,7 +32,6 @@
 
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
-import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
 import static org.hamcrest.Matchers.containsString;
@@ -71,10 +70,10 @@ public void expectMatchesAdmin(String adminSql, String user, String userSql) thr
         public void expectScrollMatchesAdmin(String adminSql, String user, String userSql) throws Exception {
             String mode = randomMode();
             Map<String, Object> adminResponse = runSql(null,
-                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + version(mode) + "}",
+                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + "}",
                             ContentType.APPLICATION_JSON), mode);
             Map<String, Object> otherResponse = runSql(user,
-                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + version(mode) + "}",
+                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + "}",
                             ContentType.APPLICATION_JSON), mode);
 
             String adminCursor = (String) adminResponse.remove("cursor");
@@ -84,11 +83,9 @@ public void expectScrollMatchesAdmin(String adminSql, String user, String userSq
             assertResponse(adminResponse, otherResponse);
             while (true) {
                 adminResponse = runSql(null,
-                        new StringEntity("{\"cursor\": \"" + adminCursor + "\"" + mode(mode) + version(mode) + "}",
-                            ContentType.APPLICATION_JSON), mode);
+                        new StringEntity("{\"cursor\": \"" + adminCursor + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), mode);
                 otherResponse = runSql(user,
-                        new StringEntity("{\"cursor\": \"" + otherCursor + "\"" + mode(mode) + version(mode) + "}",
-                            ContentType.APPLICATION_JSON), mode);
+                        new StringEntity("{\"cursor\": \"" + otherCursor + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), mode);
                 adminCursor = (String) adminResponse.remove("cursor");
                 otherCursor = (String) otherResponse.remove("cursor");
                 assertResponse(adminResponse, otherResponse);
@@ -183,8 +180,7 @@ public void checkNoMonitorMain(String user) throws Exception {
         }
 
         private static Map<String, Object> runSql(@Nullable String asUser, String mode, String sql) throws IOException {
-            return runSql(asUser, new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + version(mode) + "}",
-                ContentType.APPLICATION_JSON), mode);
+            return runSql(asUser, new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), mode);
         }
 
         private static Map<String, Object> runSql(@Nullable String asUser, HttpEntity entity, String mode) throws IOException {
@@ -234,18 +230,18 @@ protected AuditLogAsserter createAuditLogAsserter() {
      */
     public void testHijackScrollFails() throws Exception {
         createUser("full_access", "rest_minimal");
-        final String mode = randomMode();
 
+        String mode = randomMode();
         Map<String, Object> adminResponse = RestActions.runSql(null,
-                new StringEntity("{\"query\": \"SELECT * FROM test\", \"fetch_size\": 1" + mode(mode) + version(mode) + "}",
+                new StringEntity("{\"query\": \"SELECT * FROM test\", \"fetch_size\": 1" + mode(mode) + "}",
                         ContentType.APPLICATION_JSON), mode);
 
         String cursor = (String) adminResponse.remove("cursor");
         assertNotNull(cursor);
 
+        final String m = randomMode();
         ResponseException e = expectThrows(ResponseException.class, () -> RestActions.runSql("full_access",
-                new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) + "}", ContentType.APPLICATION_JSON),
-                mode));
+                new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(m) + "}", ContentType.APPLICATION_JSON), m));
         // TODO return a better error message for bad scrolls
         assertThat(e.getMessage(), containsString("No search context found for id"));
         assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
diff --git a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
index 96dfcec9150c0..e2f14ffed6c40 100644
--- a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
@@ -33,7 +33,6 @@
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.toMap;
-import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
 
@@ -178,8 +177,7 @@ private Map<String, Object> runSql(String asUser, String mode, String sql) throw
             options.addHeader("es-security-runas-user", asUser);
             request.setOptions(options);
         }
-        request.setEntity(new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + version(mode) + "}",
-            ContentType.APPLICATION_JSON));
+        request.setEntity(new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON));
         return toMap(client().performRequest(request), mode);
     }
     
diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
index cf8849bcab573..3765c45562153 100644
--- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
@@ -42,16 +42,16 @@ public void testErrorMessageForTranslatingSQLCommandStatement() throws IOExcepti
 
     public void testErrorMessageForInvalidParamDataType() throws IOException {
         expectBadRequest(() -> runTranslateSql(
-            "{\"query\":\"SELECT null WHERE 0 = ? \"" + mode("odbc") + version("odbc") +
-                ", \"params\":[{\"type\":\"invalid\", \"value\":\"irrelevant\"}]}"),
+            "{\"query\":\"SELECT null WHERE 0 = ? \", \"mode\": \"odbc\", \"params\":[{\"type\":\"invalid\", \"value\":\"irrelevant\"}]}"
+            ),
             containsString("Invalid parameter data type [invalid]")
         );
     }
 
     public void testErrorMessageForInvalidParamSpec() throws IOException {
         expectBadRequest(() -> runTranslateSql(
-            "{\"query\":\"SELECT null WHERE 0 = ? \"" + mode("odbc") + version("odbc") +
-                ", \"params\":[{\"type\":\"SHAPE\", \"value\":false}]}"),
+            "{\"query\":\"SELECT null WHERE 0 = ? \", \"mode\": \"odbc\", \"params\":[{\"type\":\"SHAPE\", \"value\":false}]}"
+            ),
             containsString("Cannot cast value [false] of type [BOOLEAN] to parameter type [SHAPE]")
         );
     }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
index 150db29a2f58e..4a760b97cd23c 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
@@ -103,40 +103,6 @@ public void testKeywordField() throws IOException {
         assertResponse(expected, runSql("SELECT keyword_field FROM test"));
     }
 
-    /*
-     *    "constant_keyword_field": {
-     *       "type": "constant_keyword",
-     *       "value": "foo"
-     *    }
-     */
-    public void testConstantKeywordField() throws IOException {
-        String value = randomAlphaOfLength(20);
-        // _source for `constant_keyword` fields doesn't matter, as they should be taken from docvalue_fields
-        boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting
-        boolean enableSource = randomBoolean();          // enable _source at index level
-        
-        Map<String, Object> indexProps = new HashMap<>(1);
-        indexProps.put("_source", enableSource);
-        
-        Map<String, Map<String, Object>> fieldProps = null;
-        if (randomBoolean()) {
-            fieldProps = new HashMap<>(1);
-            Map<String, Object> fieldProp = new HashMap<>(1);
-            fieldProp.put("value", value);
-            fieldProps.put("constant_keyword_field", fieldProp);
-        }
-        
-        createIndexWithFieldTypeAndProperties("constant_keyword", fieldProps, explicitSourceSetting ? indexProps : null);
-        index("{\"constant_keyword_field\":\"" + value + "\"}");
-        
-        Map<String, Object> expected = new HashMap<>();
-        expected.put("columns", Arrays.asList(
-                columnInfo("plain", "constant_keyword_field", "constant_keyword", JDBCType.VARCHAR, Integer.MAX_VALUE)
-        ));
-        expected.put("rows", singletonList(singletonList(value)));
-        assertResponse(expected, runSql("SELECT constant_keyword_field FROM test"));
-    }
-
     /*
      *    "long/integer/short/byte_field": {
      *       "type": "long/integer/short/byte"
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java
index 4a3b779cae0f5..d35f93f471884 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java
@@ -31,7 +31,6 @@
 import static org.elasticsearch.xpack.sql.proto.Mode.CLI;
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS;
-import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
 
 public abstract class SqlProtocolTestCase extends ESRestTestCase {
@@ -136,7 +135,7 @@ private void assertFloatingPointNumbersReturnTypes(Request request, Mode mode) t
                 + "CAST(-1234.34 AS REAL) AS float_negative,"
                 + "1234567890123.34 AS double_positive,"
                 + "-1234567890123.34 AS double_negative\""
-                + mode(mode.toString()) + version(mode.toString()) + "}";
+                + mode(mode.toString()) + "}";
         request.setEntity(new StringEntity(requestContent, ContentType.APPLICATION_JSON));
         
         Map<String, Object> map;
@@ -189,7 +188,7 @@ private void assertQuery(String sql, String columnName, String columnType, Objec
     private void assertQuery(String sql, String columnName, String columnType, Object columnValue, int displaySize, Mode mode)
             throws IOException {
         boolean columnar = randomBoolean();
-        Map<String, Object> response = runSql(mode, sql, columnar);
+        Map<String, Object> response = runSql(mode.toString(), sql, columnar);
         List<Object> columns = (ArrayList<Object>) response.get("columns");
         assertEquals(1, columns.size());
 
@@ -216,12 +215,12 @@ private void assertQuery(String sql, String columnName, String columnType, Objec
             assertEquals(columnValue, row.get(0));
         }
     }
-
-    private Map<String, Object> runSql(Mode mode, String sql, boolean columnar) throws IOException {
+    
+    private Map<String, Object> runSql(String mode, String sql, boolean columnar) throws IOException {
         Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
-        String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode.toString()) + version(mode.toString()) + "}";
+        String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode) + "}";
         String format = randomFrom(XContentType.values()).name().toLowerCase(Locale.ROOT);
-
+        
         // add a client_id to the request
         if (randomBoolean()) {
             String clientId = randomFrom(randomFrom(CLIENT_IDS), randomAlphaOfLengthBetween(10, 20));
@@ -253,13 +252,14 @@ private Map<String, Object> runSql(Mode mode, String sql, boolean columnar) thro
 
         // randomize binary response enforcement for drivers (ODBC/JDBC) and CLI
         boolean binaryCommunication = randomBoolean();
+        Mode m = Mode.fromString(mode);
         if (randomBoolean()) {
             // set it explicitly or leave the default (null) as is
             requestContent = new StringBuilder(requestContent)
                     .insert(requestContent.length() - 1, ",\"binary_format\":" + binaryCommunication).toString();
-            binaryCommunication = Mode.isDedicatedClient(mode) && binaryCommunication;
+            binaryCommunication = ((Mode.isDriver(m) || m == Mode.CLI) && binaryCommunication);
         } else {
-            binaryCommunication = Mode.isDedicatedClient(mode);
+            binaryCommunication = Mode.isDriver(m) || m == Mode.CLI;
         }
         
         // send the query either as body or as request parameter
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
index e362349f332d3..d2dffe21e1013 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
@@ -123,11 +123,13 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                 createString("first_name", createIndex);
                 createString("last_name", createIndex);
                 createIndex.startObject("gender").field("type", "keyword");
+                if (extraFields) {
+                    createIndex.field("copy_to", "extra_gender");
+                }
                 createIndex.endObject();
 
                 if (extraFields) {
-                    createIndex.startObject("extra_gender").field("type", "constant_keyword").endObject();
-                    createIndex.startObject("null_constant").field("type", "constant_keyword").endObject();
+                    createIndex.startObject("extra_gender").field("type", "keyword").endObject();
                     createIndex.startObject("extra.info.gender")
                         .field("type", "alias")
                         .field("path", "gender")
@@ -196,9 +198,6 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                     }
                     hadLastItem = true;
                     bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"');
-                    if (titles.get(f).equals("gender") && extraFields) {
-                        bulk.append(",\"extra_gender\":\"Female\"");
-                    }
                 }
             }
             // append department
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java
index b2260ba29085b..9c30d80249f75 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.sql.qa.rest;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -39,14 +38,6 @@ public static String mode(String mode) {
         return Strings.isEmpty(mode) ? StringUtils.EMPTY : ",\"mode\":\"" + mode + "\"";
     }
 
-    public static String version(String mode) {
-        Mode m = Mode.fromString(mode);
-        if (Mode.isDedicatedClient(m)) {
-            return ",\"version\":" + "\"" + Version.CURRENT.toString() + "\"";
-        }
-        return StringUtils.EMPTY;
-    }
-
     public static String randomMode() {
         return randomFrom(StringUtils.EMPTY, "jdbc", "plain");
     }
@@ -58,7 +49,7 @@ public static String randomMode() {
     public static Number xContentDependentFloatingNumberValue(String mode, Number value) {
         Mode m = Mode.fromString(mode);
         // for drivers and the CLI return the number as is, while for REST cast it implicitly to Double (the JSON standard).
-        if (Mode.isDedicatedClient(m)) {
+        if (Mode.isDriver(m) || m == Mode.CLI) {
             return value;
         } else {
             return value.doubleValue();
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
index 0f656dcfcd6e9..946a1a9a8cac6 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
@@ -105,9 +105,8 @@ public void testNextPage() throws IOException {
                 + "   SELECT text, number, SQRT(number) AS s, SCORE()"
                 + "     FROM test"
                 + " ORDER BY number, SCORE()\", "
-                + "\"mode\":\"" + mode + "\""
-                + version(mode)
-                + ", \"fetch_size\":2" + columnarParameter(columnar) + "}";
+                + "\"mode\":\"" + mode + "\", "
+            + "\"fetch_size\":2" + columnarParameter(columnar) + "}";
 
         Number value = xContentDependentFloatingNumberValue(mode, 1f);
         String cursor = null;
@@ -117,8 +116,7 @@ public void testNextPage() throws IOException {
                 response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "", mode);
             } else {
                 columnar = randomBoolean();
-                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) +
-                        columnarParameter(columnar) + "}",
+                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
                         ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode);
             }
 
@@ -153,8 +151,7 @@ public void testNextPage() throws IOException {
         } else {
             expected.put("rows", emptyList());
         }
-        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) +
-                columnarParameter(columnar) + "}",
+        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
                 ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
@@ -190,9 +187,8 @@ public void testNextPageWithDatetimeAndTimezoneParam() throws IOException {
         String sqlRequest =
                 "{\"query\":\"SELECT DATE_PART('TZOFFSET', date) AS tz FROM test_date_timezone ORDER BY date\","
                         + "\"time_zone\":\"" + zoneId.getId() + "\", "
-                        + "\"mode\":\"" + mode + "\""
-                        + version(mode)
-                        + ",\"fetch_size\":2}";
+                        + "\"mode\":\"" + mode + "\", "
+                        + "\"fetch_size\":2}";
 
         String cursor = null;
         for (int i = 0; i <= datetimes.length; i += 2) {
@@ -203,7 +199,7 @@ public void testNextPageWithDatetimeAndTimezoneParam() throws IOException {
                 expected.put("columns", singletonList(columnInfo(mode, "tz", "integer", JDBCType.INTEGER, 11)));
                 response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "", mode);
             } else {
-                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) + "}",
+                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
                         ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode);
             }
 
@@ -219,7 +215,7 @@ public void testNextPageWithDatetimeAndTimezoneParam() throws IOException {
         }
         Map<String, Object> expected = new HashMap<>();
         expected.put("rows", emptyList());
-        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) + "}",
+        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
                 ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
@@ -427,10 +423,9 @@ public void testUseColumnarForUnsupportedFormats() throws Exception {
     
     public void testUseColumnarForTranslateRequest() throws IOException {
         index("{\"test\":\"test\"}", "{\"test\":\"test\"}");
-
-        String mode = randomMode();
+        
         Request request = new Request("POST", SQL_TRANSLATE_REST_ENDPOINT);
-        request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(mode) + version(mode) + "}",
+        request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
                 ContentType.APPLICATION_JSON));
         expectBadRequest(() -> {
                 client().performRequest(request);
@@ -469,7 +464,7 @@ private Map<String, Object> runSql(String mode, String sql, boolean columnar) th
 
     private Map<String, Object> runSql(String mode, String sql, String suffix, boolean columnar) throws IOException {
         // put an explicit "columnar": false parameter or omit it altogether, it should make no difference
-        return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + version(mode) + columnarParameter(columnar) + "}",
+        return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + columnarParameter(columnar) + "}",
                 ContentType.APPLICATION_JSON), suffix, mode);
     }
     
@@ -572,10 +567,9 @@ private void executeAndAssertPrettyPrinting(String expectedJson, String prettyPa
             options.addHeader("Accept", randomFrom("*/*", "application/json"));
             request.setOptions(options);
         }
-        request.setEntity(new StringEntity("{\"query\":\"SELECT * FROM test\"" + mode("plain") + version("plain") +
-                columnarParameter(columnar) + "}",
-                ContentType.APPLICATION_JSON));
-
+        request.setEntity(new StringEntity("{\"query\":\"SELECT * FROM test\"" + mode("plain") + columnarParameter(columnar) + "}",
+                  ContentType.APPLICATION_JSON));
+        
         Response response = client().performRequest(request);
         try (InputStream content = response.getEntity().getContent()) {
             String actualJson = new BytesArray(content.readAllBytes()).utf8ToString();
@@ -604,7 +598,7 @@ public void testBasicQueryWithFilter() throws IOException {
         expected.put("columns", singletonList(columnInfo(mode, "test", "text", JDBCType.VARCHAR, Integer.MAX_VALUE)));
         expected.put("rows", singletonList(singletonList("foo")));
         assertResponse(expected, runSql(new StringEntity("{\"query\":\"SELECT * FROM test\", " +
-                "\"filter\":{\"match\": {\"test\": \"foo\"}}" + mode(mode) + version(mode) + "}",
+                "\"filter\":{\"match\": {\"test\": \"foo\"}}" + mode(mode) + "}",
                 ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
@@ -629,7 +623,7 @@ public void testBasicQueryWithParameters() throws IOException {
             "10, \"foo\"";
         assertResponse(expected, runSql(new StringEntity("{\"query\":\"SELECT test, ? param FROM test WHERE test = ?\", " +
                 "\"params\":[" + params + "]"
-                + mode(mode) + version(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
+                + mode(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
     public void testBasicTranslateQueryWithFilter() throws IOException {
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
index fe3dd9019a4ae..4d864961e8ecd 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
@@ -29,7 +29,6 @@
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_STATS_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_TRANSLATE_REST_ENDPOINT;
-import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
 
 public abstract class RestSqlUsageTestCase extends ESRestTestCase {
@@ -256,18 +255,18 @@ private void runTranslate(String sql) throws IOException {
     }
     
     private void runSql(String sql) throws IOException {
-        Mode mode = Mode.PLAIN;
+        String mode = Mode.PLAIN.toString();
         if (clientType.equals(ClientType.JDBC.toString())) {
-            mode = Mode.JDBC;
+            mode = Mode.JDBC.toString();
         } else if (clientType.startsWith(ClientType.ODBC.toString())) {
-            mode = Mode.ODBC;
+            mode = Mode.ODBC.toString();
         } else if (clientType.equals(ClientType.CLI.toString())) {
-            mode = Mode.CLI;
+            mode = Mode.CLI.toString();
         }
 
-        runSql(mode.toString(), clientType, sql);
+        runSql(mode, clientType, sql);
     }
-
+    
     @SuppressWarnings({ "unchecked", "rawtypes" })
     private void assertTranslateQueryMetric(int expected, Map<String, Object> responseAsMap) throws IOException {
         List<Map<String, Map<String, Map>>> nodesListStats = (List) responseAsMap.get("stats");
@@ -279,7 +278,7 @@ private void assertTranslateQueryMetric(int expected, Map<String, Object> respon
         }
         assertEquals(expected, actualMetricValue);
     }
-
+    
     private void runSql(String mode, String restClient, String sql) throws IOException {
         Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
         request.addParameter("error_trace", "true");   // Helps with debugging in case something crazy happens on the server.
@@ -294,8 +293,9 @@ private void runSql(String mode, String restClient, String sql) throws IOExcepti
             options.addHeader("Accept", randomFrom("*/*", "application/json"));
             request.setOptions(options);
         }
-        request.setEntity(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + version(mode) +
-            (ignoreClientType ? StringUtils.EMPTY : ",\"client_id\":\"" + restClient + "\"") + "}", ContentType.APPLICATION_JSON));
+        request.setEntity(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) +
+                (ignoreClientType ? StringUtils.EMPTY : ",\"client_id\":\"" + restClient + "\"") + "}",
+                ContentType.APPLICATION_JSON));
         client().performRequest(request);
     }
     
diff --git a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
index 440434796a163..6c51bbf3dc69b 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
@@ -41,7 +41,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |constant_keyword
+extra_gender        |VARCHAR        |keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -50,7 +50,6 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
-null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
@@ -70,7 +69,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |constant_keyword
+extra_gender        |VARCHAR        |keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -79,7 +78,6 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
-null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
index c8ea838745c46..b17ba988472be 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
@@ -276,7 +276,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |constant_keyword
+extra_gender        |VARCHAR        |keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -285,7 +285,6 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
-null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
@@ -305,7 +304,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |constant_keyword
+extra_gender        |VARCHAR        |keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -314,7 +313,6 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
-null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
deleted file mode 100644
index 42257a8890ebd..0000000000000
--- a/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
+++ /dev/null
@@ -1,163 +0,0 @@
-// To mute tests follow example in file: example.csv-spec
-
-//
-// Tests testing constant_keyword (introduced in ES 7.7)
-//
-
-// filtering
-
-filterEquals
-SELECT extra_gender g FROM "test_emp_copy" WHERE g = 'Female' LIMIT 5;
-
-    g     
----------------
-Female              
-Female              
-Female             
-Female         
-Female
-;
-
-filterNotEquals
-SELECT COUNT(*) AS c FROM "test_emp_copy" WHERE extra_gender <> 'Female';
-
-    c:l     
----------------
-0
-;
-
-aggWithNullFilter
-SELECT COUNT(*) count FROM test_emp_copy WHERE extra_gender IS NOT NULL;
-
-     count:l     
----------------
-100
-;
-
-// Awaits for https://github.com/elastic/elasticsearch/issues/53545
-aggWithNullFilter-Ignore
-SELECT COUNT(*) count FROM test_emp_copy WHERE null_constant IS NULL;
-
-     count:l     
----------------
-100
-;
-
-functionOverAlias
-SELECT BIT_LENGTH(extra_gender) bit FROM test_emp_copy ORDER BY extra_gender LIMIT 1;
-
-      bit      
----------------
-48
-;
-
-
-singlePercentileWithoutComma
-SELECT extra_gender AS gender, PERCENTILE(emp_no, 97) p1 FROM test_emp_copy GROUP BY extra_gender;
-
-gender:s             | p1:d
-Female               |10097.5        
-;
-
-percentileRank
-SELECT extra_gender AS gender, PERCENTILE_RANK(emp_no, 10025) rank FROM test_emp_copy GROUP BY extra_gender;
-
-gender:s       | rank:d
----------------+---------------
-Female         |24.5           
-;
-
-aggSumWithAliasWithColumnRepeatedWithOrderDesc
-SELECT extra_gender AS g, extra_gender, SUM(salary) AS s3, SUM(salary), SUM(salary) AS s5 FROM test_emp_copy GROUP BY extra_gender;
-
-       g:s     | extra_gender:s |      s3:i     | SUM(salary):i |      s5:i       
----------------+----------------+---------------+---------------+---------------
-Female         |Female          |4824855        |4824855        |4824855      
-;
-
-topHitsAsMinAndMaxAndGroupBy
-schema::extra_gender:s|min:s|max:s|first:s|last:s
-SELECT extra_gender, MIN(first_name) as min, MAX(first_name) as max, FIRST(first_name) as first, LAST(first_name) as last FROM test_emp_copy GROUP BY extra_gender ORDER BY extra_gender;
-
- extra_gender  |      min      |      max      |     first     |     last      
----------------+---------------+---------------+---------------+---------------
-Female         |Alejandro      |Zvonko         |Alejandro      |Zvonko         
-;
-
-medianAbsoluteDeviationOnTwoFields
-schema::extra_gender:s|avg:l|mad_s:l|mad_l:d
-SELECT extra_gender, FLOOR(AVG(salary)) AS avg, FLOOR(MAD(salary)) AS mad_s, MAD(languages) AS mad_l FROM test_emp_copy GROUP BY extra_gender ORDER BY extra_gender;
-
- extra_gender  |      avg    |     mad_s   |     mad_l     
----------------+-------------+-------------+---------------
-Female         |48248        |10096        |1            
-;
-
-caseGroupByAndHaving
-schema::count:l|extra_gender:s|languages:byte
-SELECT count(*) AS count, extra_gender, languages FROM test_emp_copy
-GROUP BY 2, 3 HAVING CASE WHEN count(*) > 10 THEN 'many' ELSE 'a few' END = 'many'
-ORDER BY 1, 3;
-
-     count     | extra_gender  |   languages   
----------------+---------------+---------------
-15             |Female         |1              
-17             |Female         |3              
-18             |Female         |4              
-19             |Female         |2              
-21             |Female         |5              
-;
-
-whereFieldWithRLikeAndGroupByOrderBy
-SELECT last_name l, extra_gender g, COUNT(*) c, MAX(salary) AS sal FROM test_emp_copy WHERE emp_no < 10050 AND (last_name RLIKE 'B.*' OR extra_gender = 'F') GROUP BY g, l ORDER BY sal;
-
-      l:s      |      g:s      |      c:l      |     sal:i      
----------------+---------------+---------------+---------------
-Berztiss       |Female         |1              |28336        
-Brender        |Female         |1              |36051        
-Bridgland      |Female         |1              |48942        
-Bouloucos      |Female         |1              |58715        
-Bamford        |Female         |1              |61805        
-;
-
-multipleGroupingsAndOrderingByGroupsWithFunctions_2
-SELECT first_name f, last_name l, LCASE(extra_gender) g, CONCAT(UCASE(first_name), LCASE(last_name)) c FROM test_emp_copy GROUP BY f, LCASE(extra_gender), l, c ORDER BY c DESC, first_name, l ASC, g LIMIT 3;
-
-      f:s      |      l:s      |      g:s      |       c:s       
----------------+---------------+---------------+---------------
-null           |Swan           |female         |swan           
-null           |Reistad        |female         |reistad        
-null           |Portugali      |female         |portugali      
-;
-
-isNotNullWithCount
-SELECT extra_gender IS NOT NULL AS bool, COUNT(*) FROM test_emp_copy GROUP BY extra_gender;
-
-          bool:b        |   COUNT(*):l    
-------------------------+---------------
-true                    |100            
-;
-
-likeWithCount1
-SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender LIKE 'F%';
-
-   COUNT(*):l    
----------------
-100            
-;
-
-likeWithCount2
-SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender LIKE '%m%';
-
-   COUNT(*):l    
----------------
-100            
-;
-
-likeWithCount3
-SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender NOT LIKE '%m%';
-
-   COUNT(*):l    
----------------
-0         
-;
\ No newline at end of file
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
index dee74303b9bda..711df01849103 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
@@ -29,20 +29,19 @@ SYS COLUMNS TABLE LIKE 'test_emp';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest  |null           |test_emp            |birth_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |emp_no             |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra.info.gender  |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra_gender       |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra_no           |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |first_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |first_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |gender             |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |hire_date          |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |languages          |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |last_name          |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |last_name.keyword  |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
-integTest  |null           |test_emp            |null_constant      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |salary             |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsWithCatalogAndLike
@@ -50,20 +49,19 @@ SYS COLUMNS CATALOG 'integTest' TABLE LIKE 'test\_emp\_copy' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s  |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest  |null           |test_emp_copy|birth_date        |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|emp_no            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra_no          |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|first_name        |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|gender            |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|hire_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|languages         |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|last_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
-integTest  |null           |test_emp_copy|null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|salary            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsOnAliasWithTableLike
@@ -71,57 +69,55 @@ SYS COLUMNS TABLE LIKE 'test\_alias' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest   |null           |test_alias    |birth_date        |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |emp_no            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra_no          |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |first_name        |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |gender            |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |hire_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |languages         |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |last_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
-integTest   |null           |test_alias    |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |salary            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsAllTables
 SYS COLUMNS TABLE LIKE '%';
 
-   TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s  |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
----------------+---------------+---------------+------------------+---------------+----------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------
-integTest      |null           |logs           |@timestamp        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_in          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |2                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_out         |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_ip         |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_port       |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |dest_ip           |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |id                |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |status            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_no          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |12                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |13                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |14                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |15                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |16                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |17                |YES          |null           |null           |null           |null            |NO                |NO
+   TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
+---------------+---------------+---------------+------------------+---------------+---------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------
+integTest      |null           |logs           |@timestamp        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |bytes_in          |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |2                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |bytes_out         |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |client_ip         |12             |IP             |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |4                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |client_port       |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |5                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |dest_ip           |12             |IP             |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |id                |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |status            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |birth_date        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |emp_no            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |first_name        |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |first_name.keyword|12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |gender            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |hire_date         |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |languages         |-6             |BYTE           |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |last_name         |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |last_name.keyword |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |salary            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |11                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |birth_date        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |emp_no            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra_gender      |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra_no          |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |first_name        |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |gender            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |11                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |hire_date         |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |12                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |languages         |-6             |BYTE           |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |13                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |last_name         |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |14                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |15                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |salary            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |16                |YES          |null           |null           |null           |null            |NO                |NO
 ;
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index fe30b9975cab4..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
new file mode 100644
index 0000000000000..3fd9e819ce181
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
@@ -0,0 +1 @@
+ca406661129d35008411365d2b6e747dc39378af
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
index d0260551c1c88..27712e93205de 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
@@ -5,12 +5,9 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
@@ -36,8 +33,6 @@
 import java.util.Objects;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
 /**
  * Base class for requests that contain sql queries (Query and Translate)
  */
@@ -51,8 +46,7 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme
     @Nullable
     private QueryBuilder filter = null;
     private List<SqlTypedParamValue> params = Collections.emptyList();
-
-    // TODO: define all REST request object field names in a protocol class as unique source
+    
     static final ParseField QUERY = new ParseField("query");
     static final ParseField CURSOR = new ParseField("cursor");
     static final ParseField PARAMS = new ParseField("params");
@@ -63,7 +57,6 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme
     static final ParseField FILTER = new ParseField("filter");
     static final ParseField MODE = new ParseField("mode");
     static final ParseField CLIENT_ID = new ParseField("client_id");
-    static final ParseField CLIENT_VERSION = new ParseField("version");
 
     public AbstractSqlQueryRequest() {
         super();
@@ -87,8 +80,7 @@ protected static <R extends AbstractSqlQueryRequest> ObjectParser<R, Void> objec
         ObjectParser<R, Void> parser = new ObjectParser<>("sql/query", false, supplier);
         parser.declareString(AbstractSqlQueryRequest::query, QUERY);
         parser.declareString((request, mode) -> request.mode(Mode.fromString(mode)), MODE);
-        parser.declareString(AbstractSqlRequest::clientId, CLIENT_ID);
-        parser.declareString(AbstractSqlRequest::version, CLIENT_VERSION);
+        parser.declareString((request, clientId) -> request.clientId(clientId), CLIENT_ID);
         parser.declareField(AbstractSqlQueryRequest::params, AbstractSqlQueryRequest::parseParams, PARAMS, ValueType.VALUE_ARRAY);
         parser.declareString((request, zoneId) -> request.zoneId(ZoneId.of(zoneId)), TIME_ZONE);
         parser.declareInt(AbstractSqlQueryRequest::fetchSize, FETCH_SIZE);
@@ -213,26 +205,6 @@ protected static void validateParams(List<SqlTypedParamValue> params, Mode mode)
         }
     }
 
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        // the version field is mandatory for drivers and CLI
-        Mode mode = requestInfo().mode();
-        if (mode != null && (Mode.isDedicatedClient(mode))) {
-            if (requestInfo().version() == null) {
-                if (Strings.hasText(query())) {
-                    validationException = addValidationError("[version] is required for the [" + mode.toString() + "] client",
-                        validationException);
-                }
-            } else if (requestInfo().version().equals(Version.CURRENT.toString()) == false) {
-                validationException = addValidationError("The [" + requestInfo().version() + "] version of the [" +
-                        mode.toString() + "] " + "client is not compatible with Elasticsearch version [" + Version.CURRENT + "]",
-                    validationException);
-            }
-        }
-        return validationException;
-    }
-
     /**
      * The client's time zone
      */
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java
index a9fb2f0509fef..f5965cc0e681f 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.xpack.sql.proto.Mode;
 import org.elasticsearch.xpack.sql.proto.RequestInfo;
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -40,8 +39,7 @@ protected AbstractSqlRequest(StreamInput in) throws IOException {
         super(in);
         Mode mode = in.readEnum(Mode.class);
         String clientId = in.readOptionalString();
-        String clientVersion = in.readOptionalString();
-        requestInfo = new RequestInfo(mode, clientId, clientVersion);
+        requestInfo = new RequestInfo(mode, clientId);
     }
 
     @Override
@@ -58,9 +56,8 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeEnum(requestInfo.mode());
         out.writeOptionalString(requestInfo.clientId());
-        out.writeOptionalString(requestInfo.version() == null ? null : requestInfo.version().toString());
     }
-
+    
     public RequestInfo requestInfo() {
         return requestInfo;
     }
@@ -89,14 +86,6 @@ public void clientId(String clientId) {
         this.requestInfo.clientId(clientId);
     }
 
-    public void version(String clientVersion) {
-        requestInfo.version(clientVersion);
-    }
-
-    public SqlVersion version() {
-        return requestInfo.version();
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java
index 5b2974d84fb4d..563b1fe7cf886 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java
@@ -21,7 +21,6 @@
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CLIENT_ID;
-import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CLIENT_VERSION;
 import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CURSOR;
 import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.MODE;
 
@@ -43,7 +42,6 @@ public class SqlClearCursorRequest extends AbstractSqlRequest {
         PARSER.declareString(constructorArg(), CURSOR);
         PARSER.declareString(optionalConstructorArg(), MODE);
         PARSER.declareString(optionalConstructorArg(), CLIENT_ID);
-        PARSER.declareString(optionalConstructorArg(), CLIENT_VERSION);
     }
 
     private String cursor;
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
index 4329c5101c8ee..a338410b7278e 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
@@ -72,7 +72,7 @@ public SqlQueryRequest(String query, List<SqlTypedParamValue> params, QueryBuild
 
     @Override
     public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = super.validate();
+        ActionRequestValidationException validationException = null;
         if ((false == Strings.hasText(query())) && Strings.hasText(cursor) == false) {
             validationException = addValidationError("one of [query] or [cursor] is required", validationException);
         }
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java
index 2e3f39c8ff449..f2ede4b463cdf 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java
@@ -52,11 +52,6 @@ public SqlQueryRequestBuilder mode(Mode mode) {
         return this;
     }
 
-    public SqlQueryRequestBuilder version(String version) {
-        request.version(version);
-        return this;
-    }
-
     public SqlQueryRequestBuilder cursor(String cursor) {
         request.cursor(cursor);
         return this;
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java
index 5b4a4227980dc..708779ec6e3c3 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java
@@ -44,7 +44,7 @@ public SqlTranslateRequest(StreamInput in) throws IOException {
 
     @Override
     public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = super.validate();
+        ActionRequestValidationException validationException = null;
         if ((false == Strings.hasText(query()))) {
             validationException = addValidationError("query is required", validationException);
         }
@@ -65,12 +65,12 @@ public static SqlTranslateRequest fromXContent(XContentParser parser) {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         // This is needed just to test parsing of SqlTranslateRequest, so we can reuse SqlQuerySerialization
-        return new SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(), pageTimeout(),
-            filter(),
-            null,
-            null,
+        return new SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(), pageTimeout(), 
+            filter(), 
+            null, 
+            null, 
             requestInfo(),
-            false,
+            false, 
             false,
             null).toXContent(builder, params);
     }
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
index 020d0c18d52ac..d902e75121c75 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
@@ -176,9 +176,6 @@ private static void toXContent(SqlQueryRequest request, XContentBuilder builder)
         if (request.clientId() != null) {
             builder.field("client_id", request.clientId());
         }
-        if (request.version() != null) {
-            builder.field("version", request.version().toString());
-        }
         if (request.params() != null && request.params().isEmpty() == false) {
             builder.startArray("params");
             for (SqlTypedParamValue val : request.params()) {
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
index 4211324e41491..34245945e506b 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -49,10 +48,9 @@ public void testClearCursorRequestParser() throws IOException {
         Mode randomMode = randomFrom(Mode.values());
         
         SqlClearCursorRequest request = generateRequest("{\"cursor\" : \"whatever\", \"mode\" : \""
-                + randomMode.toString() + "\", \"client_id\" : \"bla\", \"version\": \"1.2.3\"}",
+                + randomMode.toString() + "\", \"client_id\" : \"bla\"}",
                 SqlClearCursorRequest::fromXContent);
         assertNull(request.clientId());
-        assertNull(request.version());
         assertEquals(randomMode, request.mode());
         assertEquals("whatever", request.getCursor());
         
@@ -68,11 +66,10 @@ public void testClearCursorRequestParser() throws IOException {
         assertNull(request.clientId());
         assertEquals(Mode.PLAIN, request.mode());
         assertEquals("whatever", request.getCursor());
-
-        request = generateRequest("{\"cursor\" : \"whatever\", \"client_id\" : \"CLI\", \"version\": \"1.2.3\"}",
+        
+        request = generateRequest("{\"cursor\" : \"whatever\", \"client_id\" : \"CLI\"}",
                 SqlClearCursorRequest::fromXContent);
         assertNull(request.clientId());
-        assertNull(request.version());
         assertEquals(Mode.PLAIN, request.mode());
         assertEquals("whatever", request.getCursor());
         
@@ -104,8 +101,6 @@ public void testQueryRequestParser() throws IOException {
                 SqlQueryRequest::fromXContent);
         assertParsingErrorMessage("{\"client_id\":123}", "client_id doesn't support values of type: VALUE_NUMBER",
                 SqlQueryRequest::fromXContent);
-        assertParsingErrorMessage("{\"version\":123}", "version doesn't support values of type: VALUE_NUMBER",
-            SqlQueryRequest::fromXContent);
         assertParsingErrorMessage("{\"params\":[{\"value\":123}]}", "failed to parse field [params]", SqlQueryRequest::fromXContent);
         assertParsingErrorMessage("{\"time_zone\":12}", "time_zone doesn't support values of type: VALUE_NUMBER",
                 SqlQueryRequest::fromXContent);
@@ -113,10 +108,7 @@ public void testQueryRequestParser() throws IOException {
         Mode randomMode = randomFrom(Mode.values());
         String params;
         List<SqlTypedParamValue> list = new ArrayList<>(1);
-
-        final String clientVersion = Mode.isDedicatedClient(randomMode)
-            ? "\"version\": \"" + Version.CURRENT.toString() + "\","
-            : "";
+        
         if (Mode.isDriver(randomMode)) {
             params = "{\"value\":123, \"type\":\"whatever\"}";
             list.add(new SqlTypedParamValue("whatever", 123, true));
@@ -127,16 +119,12 @@ public void testQueryRequestParser() throws IOException {
         
         SqlQueryRequest request = generateRequest("{\"cursor\" : \"whatever\", \"mode\" : \""
                 + randomMode.toString() + "\", \"client_id\" : \"bla\","
-                + clientVersion
                 + "\"query\":\"select\","
                 + "\"params\":[" + params + "],"
                 + " \"time_zone\":\"UTC\","
                 + "\"request_timeout\":\"5s\",\"page_timeout\":\"10s\"}", SqlQueryRequest::fromXContent);
         assertNull(request.clientId());
         assertEquals(randomMode, request.mode());
-        if (Mode.isDedicatedClient(randomMode)) {
-            assertEquals(Version.CURRENT.toString(), request.version().toString());
-        }
         assertEquals("whatever", request.cursor());
         assertEquals("select", request.query());
 
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java
index ebf1883802515..c07a7f63031a8 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.xpack.sql.cli.command.ServerInfoCliCommand;
 import org.elasticsearch.xpack.sql.cli.command.ServerQueryCliCommand;
 import org.elasticsearch.xpack.sql.client.ClientException;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ConnectionConfiguration;
 import org.elasticsearch.xpack.sql.client.HttpClient;
+import org.elasticsearch.xpack.sql.client.Version;
 import org.jline.terminal.TerminalBuilder;
 
 import java.io.IOException;
@@ -161,7 +161,7 @@ private void checkConnection(CliSession cliSession, CliTerminal cliTerminal, Con
                 // Most likely we connected to something other than Elasticsearch
                 throw new UserException(ExitCodes.DATA_ERROR,
                         "Cannot communicate with the server " + con.connectionString() +
-                                ". This version of CLI only works with Elasticsearch version " + ClientVersion.CURRENT.toString());
+                                ". This version of CLI only works with Elasticsearch version " + Version.CURRENT.toString());
             }
         }
     }
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java
index 78d126f44b5bb..678fc8a9c6efb 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java
@@ -5,12 +5,11 @@
  */
 package org.elasticsearch.xpack.sql.cli.command;
 
-import org.elasticsearch.xpack.sql.client.ClientException;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.HttpClient;
+import org.elasticsearch.xpack.sql.client.ClientException;
+import org.elasticsearch.xpack.sql.client.Version;
 import org.elasticsearch.xpack.sql.proto.MainResponse;
 import org.elasticsearch.xpack.sql.proto.Protocol;
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.sql.SQLException;
 
@@ -74,11 +73,11 @@ public void checkConnection() throws ClientException {
         } catch (SQLException ex) {
             throw new ClientException(ex);
         }
-        SqlVersion version = SqlVersion.fromString(response.getVersion());
-        if (ClientVersion.isServerCompatible(version) == false) {
-            throw new ClientException("This version of the CLI is only compatible with Elasticsearch version " +
-                ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server version " +
-                version.toString());
+        Version version = Version.fromString(response.getVersion());
+        // TODO: We can relax compatibility requirement later when we have a better idea about protocol compatibility guarantees
+        if (version.major != Version.CURRENT.major || version.minor != Version.CURRENT.minor) {
+            throw new ClientException("This alpha version of CLI is only compatible with Elasticsearch version " +
+                    Version.CURRENT.toString());
         }
     }
 }
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
index bad55264768eb..40cce57321d65 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
@@ -8,7 +8,7 @@
 import org.elasticsearch.xpack.sql.cli.Cli;
 import org.elasticsearch.xpack.sql.cli.CliTerminal;
 import org.elasticsearch.xpack.sql.cli.FatalCliException;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
+import org.elasticsearch.xpack.sql.client.Version;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -53,9 +53,9 @@ public void printLogo(CliTerminal terminal) {
         }
 
         // print the version centered on the last line
-        char[] whitespaces = new char[(lineLength - ClientVersion.CURRENT.version.length()) / 2];
+        char[] whitespaces = new char[(lineLength - Version.CURRENT.version.length()) / 2];
         Arrays.fill(whitespaces, ' ');
-        terminal.println(new StringBuilder().append(whitespaces).append(ClientVersion.CURRENT.version).toString());
+        terminal.println(new StringBuilder().append(whitespaces).append(Version.CURRENT.version).toString());
         terminal.println();
     }
 
diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java
index 438f67372ff74..295ebe1fb0792 100644
--- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java
+++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java
@@ -9,11 +9,10 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.cli.command.CliSession;
-import org.elasticsearch.xpack.sql.client.ClientException;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.HttpClient;
+import org.elasticsearch.xpack.sql.client.ClientException;
+import org.elasticsearch.xpack.sql.client.Version;
 import org.elasticsearch.xpack.sql.proto.MainResponse;
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.sql.SQLException;
 
@@ -27,7 +26,7 @@ public class CliSessionTests extends ESTestCase {
 
     public void testProperConnection() throws Exception {
         HttpClient httpClient = mock(HttpClient.class);
-        when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5), ClientVersion.CURRENT.toString(),
+        when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5), org.elasticsearch.Version.CURRENT.toString(),
                 ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
         CliSession cliSession = new CliSession(httpClient);
         cliSession.checkConnection();
@@ -39,43 +38,28 @@ public void testConnection() throws Exception {
         HttpClient httpClient = mock(HttpClient.class);
         when(httpClient.serverInfo()).thenThrow(new SQLException("Cannot connect"));
         CliSession cliSession = new CliSession(httpClient);
-        Throwable throwable = expectThrows(ClientException.class, cliSession::checkConnection);
-        assertEquals("java.sql.SQLException: Cannot connect", throwable.getMessage());
+        expectThrows(ClientException.class, cliSession::checkConnection);
         verify(httpClient, times(1)).serverInfo();
         verifyNoMoreInteractions(httpClient);
     }
 
     public void testWrongServerVersion() throws Exception {
-        HttpClient httpClient = mock(HttpClient.class);
-        SqlVersion version = new SqlVersion((int)SqlVersion.V_7_7_0.major, SqlVersion.V_7_7_0.minor - 1, 0);
-        when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5), version.toString(),
-                ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
-        CliSession cliSession = new CliSession(httpClient);
-        Throwable throwable = expectThrows(ClientException.class, cliSession::checkConnection);
-        assertEquals("This version of the CLI is only compatible with Elasticsearch version " +
-            ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server version " + version.toString(),
-            throwable.getMessage());
-        verify(httpClient, times(1)).serverInfo();
-        verifyNoMoreInteractions(httpClient);
-    }
-
-    public void testHigherServerVersion() throws Exception {
         HttpClient httpClient = mock(HttpClient.class);
         byte minor;
         byte major;
         if (randomBoolean()) {
-            minor = ClientVersion.CURRENT.minor;
-            major = (byte) (ClientVersion.CURRENT.major + 1);
+            minor = Version.CURRENT.minor;
+            major = (byte) (Version.CURRENT.major + 1);
         } else {
-            minor = (byte) (ClientVersion.CURRENT.minor + 1);
-            major = ClientVersion.CURRENT.major;
+            minor = (byte) (Version.CURRENT.minor + 1);
+            major = Version.CURRENT.major;
 
         }
         when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5),
-            SqlVersion.fromString(major + "." + minor + ".23").toString(),
-            ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
+                org.elasticsearch.Version.fromString(major + "." + minor + ".23").toString(),
+                ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
         CliSession cliSession = new CliSession(httpClient);
-        cliSession.checkConnection();
+        expectThrows(ClientException.class, cliSession::checkConnection);
         verify(httpClient, times(1)).serverInfo();
         verifyNoMoreInteractions(httpClient);
     }
diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java
index 5b8687b950468..96e3e417531a7 100644
--- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java
+++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java
@@ -5,17 +5,17 @@
  */
 package org.elasticsearch.xpack.sql.cli;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
+import org.elasticsearch.xpack.sql.client.Version;
 
 public class VersionTests extends ESTestCase {
     public void testVersionIsCurrent() {
         /* This test will only work properly in gradle because in gradle we run the tests
          * using the jar. */
-        assertEquals(Version.CURRENT.major, ClientVersion.CURRENT.major);
-        assertEquals(Version.CURRENT.minor, ClientVersion.CURRENT.minor);
-        assertEquals(Version.CURRENT.revision, ClientVersion.CURRENT.revision);
+        assertNotNull(Version.CURRENT.hash);
+        assertEquals(org.elasticsearch.Version.CURRENT.major, Version.CURRENT.major);
+        assertEquals(org.elasticsearch.Version.CURRENT.minor, Version.CURRENT.minor);
+        assertEquals(org.elasticsearch.Version.CURRENT.revision, Version.CURRENT.revision);
     }
 
 }
diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
index e12889a264959..39c48bc31d926 100644
--- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
+++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
@@ -7,8 +7,8 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.cli.TestTerminal;
-import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.HttpClient;
+import org.elasticsearch.xpack.sql.client.Version;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.mockito.Mockito.mock;
@@ -94,7 +94,7 @@ public void testPrintLogo() throws Exception {
         testTerminal.print("not clean");
         assertTrue(new PrintLogoCommand().handle(testTerminal, cliSession, "logo"));
         assertThat(testTerminal.toString(), containsString("SQL"));
-        assertThat(testTerminal.toString(), containsString(ClientVersion.CURRENT.version));
+        assertThat(testTerminal.toString(), containsString(Version.CURRENT.version));
         verifyNoMoreInteractions(httpClient);
     }
 
diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java
index 18de5d4eb4e05..9d3a50b57576d 100644
--- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java
+++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java
@@ -57,6 +57,8 @@ public class ConnectionConfiguration {
 
     public static final String PAGE_SIZE = "page.size";
     private static final String PAGE_SIZE_DEFAULT = "1000";
+    
+    public static final String CLIENT_ID = "client_id";
 
     // Auth
 
diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java
index 2b2eee541472f..32630abb94370 100644
--- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java
+++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java
@@ -70,7 +70,7 @@ public SqlQueryResponse basicQuery(String query, int fetchSize) throws SQLExcept
                 null,
                 Boolean.FALSE,
                 null,
-                new RequestInfo(Mode.CLI, ClientVersion.CURRENT),
+                new RequestInfo(Mode.CLI),
                 false,
                 false,
                 cfg.binaryCommunication());
diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/Version.java
similarity index 57%
rename from x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java
rename to x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/Version.java
index 6b48dc97f2c9c..93db1b506369f 100644
--- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java
+++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/Version.java
@@ -5,8 +5,6 @@
  */
 package org.elasticsearch.xpack.sql.client;
 
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
-
 import java.io.IOException;
 import java.net.URL;
 import java.net.URLConnection;
@@ -18,25 +16,45 @@
 import java.util.jar.JarInputStream;
 import java.util.jar.Manifest;
 
-/**
- * Clients-specific version utility class.
- * <p>
- *     The class provides the SQL clients the version identifying the release they're are part of. The version is read from the
- *     encompassing JAR file (Elasticsearch-specific attribute in the manifest).
- *     The class is also a provider for the implemented JDBC standard.
- * </p>
- */
-public class ClientVersion {
+public class Version {
+
+    public static final Version CURRENT;
+    public final String version;
+    public final String hash;
+    public final byte major;
+    public final byte minor;
+    public final byte revision;
 
-    public static final SqlVersion CURRENT;
+    private Version(String version, String hash, byte... parts) {
+        this.version = version;
+        this.hash = hash;
+        this.major = parts[0];
+        this.minor = parts[1];
+        this.revision = parts[2];
+    }
+
+    public static Version fromString(String version) {
+        return new Version(version, "Unknown", from(version));
+    }
+
+    static byte[] from(String ver) {
+        String[] parts = ver.split("[.-]");
+        // Allow for optional snapshot and qualifier
+        if (parts.length < 3 || parts.length > 5) {
+            throw new IllegalArgumentException("Invalid version " + ver);
+        }
+        else {
+            return new byte[] { Byte.parseByte(parts[0]), Byte.parseByte(parts[1]), Byte.parseByte(parts[2]) };
+        }
+    }
 
     static {
         // check classpath
-        String target = ClientVersion.class.getName().replace(".", "/").concat(".class");
+        String target = Version.class.getName().replace(".", "/").concat(".class");
         Enumeration<URL> res;
 
         try {
-            res = ClientVersion.class.getClassLoader().getResources(target);
+            res = Version.class.getClassLoader().getResources(target);
         } catch (IOException ex) {
             throw new IllegalArgumentException("Cannot detect Elasticsearch JDBC jar; it typically indicates a deployment issue...");
         }
@@ -67,35 +85,41 @@ public class ClientVersion {
         }
 
         // This is similar to how Elasticsearch's Build class digs up its build information.
-        URL url = SqlVersion.class.getProtectionDomain().getCodeSource().getLocation();
+        // Since version info is not critical, the parsing is lenient
+        URL url = Version.class.getProtectionDomain().getCodeSource().getLocation();
         CURRENT = extractVersion(url);
     }
 
-    static SqlVersion extractVersion(URL url) {
+    static Version extractVersion(URL url) {
         String urlStr = url.toString();
+        byte maj = 0, min = 0, rev = 0;
+        String ver = "Unknown";
+        String hash = ver;
+        
         if (urlStr.endsWith(".jar") || urlStr.endsWith(".jar!/")) {
             try {
                 URLConnection conn = url.openConnection();
                 conn.setUseCaches(false);
-
+                
                 try (JarInputStream jar = new JarInputStream(conn.getInputStream())) {
                     Manifest manifest = jar.getManifest();
-                    String version = manifest.getMainAttributes().getValue("X-Compile-Elasticsearch-Version");
-                    return SqlVersion.fromString(version);
+                    hash = manifest.getMainAttributes().getValue("Change");
+                    ver = manifest.getMainAttributes().getValue("X-Compile-Elasticsearch-Version");
+                    byte[] vers = from(ver);
+                    maj = vers[0];
+                    min = vers[1];
+                    rev = vers[2];
                 }
             } catch (Exception ex) {
                 throw new IllegalArgumentException("Detected Elasticsearch JDBC jar but cannot retrieve its version", ex);
             }
         }
-        return new SqlVersion(0, 0, 0);
+        return new Version(ver, hash, maj, min, rev);
     }
 
-    // This function helps ensure that a client won't attempt to communicate to a server with less features than its own. Since this check
-    // is part of the client's start-up check that might not involve an actual SQL API request, the client has to do a bare version check
-    // as well.
-    public static boolean isServerCompatible(SqlVersion server) {
-        // Starting with this version, the compatibility logic moved from the client to the server.
-        return SqlVersion.hasVersionCompatibility(server);
+    @Override
+    public String toString() {
+        return "v" + version + " [" + hash + "]";
     }
 
     public static int jdbcMajorVersion() {
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java
index ea4d6a0834ca4..01ca2bd0dade1 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java
@@ -158,7 +158,7 @@ private void assertBinaryRequestForDrivers(boolean isBinary, XContentType xConte
                 null,
                 randomBoolean(),
                 randomAlphaOfLength(128),
-                new RequestInfo(mode, ClientVersion.CURRENT),
+                new RequestInfo(mode),
                 randomBoolean(),
                 randomBoolean(),
                 isBinary);
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java
index dabe7cdc0a384..03979a0a2b4e5 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.sql.client;
 
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
@@ -20,21 +19,45 @@
 import java.util.jar.Manifest;
 
 public class VersionTests extends ESTestCase {
+    public void test70Version() {
+        byte[] ver = Version.from("7.0.0-alpha");
+        assertEquals(7, ver[0]);
+        assertEquals(0, ver[1]);
+        assertEquals(0, ver[2]);
+    }
+
+    public void test712Version() {
+        byte[] ver = Version.from("7.1.2");
+        assertEquals(7, ver[0]);
+        assertEquals(1, ver[1]);
+        assertEquals(2, ver[2]);
+    }
 
     public void testCurrent() {
-        SqlVersion ver = SqlVersion.fromString(org.elasticsearch.Version.CURRENT.toString());
-        assertEquals(ver, ClientVersion.CURRENT);
+        Version ver = Version.fromString(org.elasticsearch.Version.CURRENT.toString());
+        assertEquals(org.elasticsearch.Version.CURRENT.major, ver.major);
+        assertEquals(org.elasticsearch.Version.CURRENT.minor, ver.minor);
+        assertEquals(org.elasticsearch.Version.CURRENT.revision, ver.revision);
     }
 
-    public void testInvalidVersion() {
-        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> SqlVersion.fromString("7.1"));
-        assertEquals("Invalid version format [7.1]", err.getMessage());
+    public void testFromString() {
+        Version ver = Version.fromString("1.2.3");
+        assertEquals(1, ver.major);
+        assertEquals(2, ver.minor);
+        assertEquals(3, ver.revision);
+        assertEquals("Unknown", ver.hash);
+        assertEquals("1.2.3", ver.version);
     }
 
+    public void testInvalidVersion() {
+        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> Version.from("7.1"));
+        assertEquals("Invalid version 7.1", err.getMessage());
+    }
+    
     public void testVersionFromJarInJar() throws IOException {
         final String JDBC_JAR_NAME = "es-sql-jdbc.jar";
         final String JAR_PATH_SEPARATOR = "!/";
-
+        
         Path dir = createTempDir();
         Path jarPath = dir.resolve("uberjar.jar");          // simulated uberjar containing the jdbc driver
         Path innerJarPath = dir.resolve(JDBC_JAR_NAME); // simulated ES JDBC driver file
@@ -65,11 +88,12 @@ public void testVersionFromJarInJar() throws IOException {
         }
         
         URL jarInJar = new URL("jar:" + jarPath.toUri().toURL().toString() + JAR_PATH_SEPARATOR + JDBC_JAR_NAME + JAR_PATH_SEPARATOR);
-
-        SqlVersion version = ClientVersion.extractVersion(jarInJar);
+        
+        Version version = Version.extractVersion(jarInJar);
         assertEquals(1, version.major);
         assertEquals(2, version.minor);
         assertEquals(3, version.revision);
+        assertEquals("abc", version.hash);
         assertEquals("1.2.3", version.version);
     }
 }
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java
index c6c9b8dffe0b7..9e8cc2fb08e5f 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java
@@ -34,10 +34,6 @@ public String clientId() {
         return requestInfo.clientId();
     }
 
-    public SqlVersion version() {
-        return requestInfo.version();
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java
index c5b8faed398ef..26eb4867b3509 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java
@@ -33,8 +33,4 @@ public String toString() {
     public static boolean isDriver(Mode mode) {
         return mode == JDBC || mode == ODBC;
     }
-
-    public static boolean isDedicatedClient(Mode mode) {
-        return mode == JDBC || mode == ODBC || mode == CLI;
-    }
 }
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java
index d26569684e391..97a3622a64f1f 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java
@@ -35,25 +35,14 @@ public class RequestInfo {
     
     private Mode mode;
     private String clientId;
-    private SqlVersion version;
-
+    
     public RequestInfo(Mode mode) {
-        this(mode, null, null);
+        this(mode, null);
     }
     
     public RequestInfo(Mode mode, String clientId) {
-        this(mode, clientId, null);
-    }
-
-    public RequestInfo(Mode mode, String clientId, String version) {
         mode(mode);
         clientId(clientId);
-        version(version);
-    }
-
-    public RequestInfo(Mode mode, SqlVersion version) {
-        mode(mode);
-        this.version = version;
     }
     
     public Mode mode() {
@@ -78,14 +67,6 @@ public void clientId(String clientId) {
         this.clientId = clientId;
     }
 
-    public void version(String clientVersion) {
-        this.version = SqlVersion.fromString(clientVersion);
-    }
-
-    public SqlVersion version() {
-        return version;
-    }
-
     @Override
     public int hashCode() {
         return Objects.hash(mode, clientId);
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java
index 7b7578320d6c4..b1f374e74a8da 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java
@@ -47,9 +47,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (clientId() != null) {
             builder.field("client_id", clientId());
         }
-        if (version() != null) {
-            builder.field("version", version().toString());
-        }
         return builder;
     }
 }
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java
index e3d35ad2a9dbf..e6d42c3898a91 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java
@@ -180,9 +180,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (clientId() != null) {
             builder.field("client_id", clientId());
         }
-        if (version() != null) {
-            builder.field("version", version().toString());
-        }
         if (this.params != null && this.params.isEmpty() == false) {
             builder.startArray("params");
             for (SqlTypedParamValue val : this.params) {
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java
deleted file mode 100644
index 4b311a373aa2a..0000000000000
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.sql.proto;
-
-import java.security.InvalidParameterException;
-
-/**
- * Elasticsearch's version modeler for the SQL plugin.
- * <p>
- *     The class models the version that the Elasticsearch server and the clients use to identify their release by. It is similar to
- *     server's <strong>Version</strong> class (which is unavailable in this package), specific to the SQL plugin and its clients, and an
- *     aid to establish the compatibility between them.
- * </p>
- */
-public class SqlVersion implements Comparable<SqlVersion>{
-
-    public final int id;
-    public final String version; // originally provided String representation
-    public final byte major;
-    public final byte minor;
-    public final byte revision;
-
-    public static final int REVISION_MULTIPLIER = 100;
-    public static final int MINOR_MULTIPLIER = REVISION_MULTIPLIER * REVISION_MULTIPLIER;
-    public static final int MAJOR_MULTIPLIER = REVISION_MULTIPLIER * MINOR_MULTIPLIER;
-
-    public static final SqlVersion V_7_7_0 = new SqlVersion(7, 7, 0);
-
-    public SqlVersion(byte major, byte minor, byte revision) {
-        this(toString(major, minor, revision), major, minor, revision);
-    }
-
-    public SqlVersion(Integer major, Integer minor, Integer revision) {
-        this(major.byteValue(), minor.byteValue(), revision.byteValue());
-        if (major > Byte.MAX_VALUE || minor > Byte.MAX_VALUE || revision > Byte.MAX_VALUE) {
-            throw new InvalidParameterException("Invalid version initialisers [" + major + ", " + minor + ", " + revision + "]");
-        }
-    }
-
-    protected SqlVersion(String version, byte... parts) {
-        this.version = version;
-
-        assert parts.length >= 3 : "Version must be initialized with all Major.Minor.Revision components";
-        this.major = parts[0];
-        this.minor = parts[1];
-        this.revision = parts[2];
-
-        if ((major | minor | revision) < 0 || minor >= REVISION_MULTIPLIER || revision >= REVISION_MULTIPLIER) {
-            throw new InvalidParameterException("Invalid version initialisers [" + major + ", " + minor + ", " + revision + "]");
-        }
-
-        id = Integer.valueOf(major) * MAJOR_MULTIPLIER
-            + Integer.valueOf(minor) * MINOR_MULTIPLIER
-            + Integer.valueOf(revision) * REVISION_MULTIPLIER;
-    }
-
-    public static SqlVersion fromString(String version) {
-        if (version == null || version.isEmpty()) {
-            return null;
-        }
-        return new SqlVersion(version, from(version));
-    }
-
-    protected static byte[] from(String ver) {
-        String[] parts = ver.split("[.-]");
-        // Allow for optional snapshot and qualifier (Major.Minor.Revision-Qualifier-SNAPSHOT)
-        if (parts.length >= 3 && parts.length <= 5) {
-            try {
-                return new byte[] { Byte.parseByte(parts[0]), Byte.parseByte(parts[1]), Byte.parseByte(parts[2]) };
-            } catch (NumberFormatException nfe) {
-                throw new IllegalArgumentException("Invalid version format [" + ver + "]: " + nfe.getMessage());
-            }
-        } else {
-            throw new IllegalArgumentException("Invalid version format [" + ver + "]");
-        }
-
-    }
-
-    private static String toString(byte... parts) {
-        assert parts.length >= 1 : "Version must contain at least a Major component";
-        String ver = String.valueOf(parts[0]);
-        for (int i = 1; i < parts.length; i ++) {
-            ver += "." + parts[i];
-        }
-        return ver;
-    }
-
-    @Override
-    public String toString() {
-        return toString(major, minor, revision);
-    }
-
-    public String majorMinorToString() {
-        return toString(major, minor);
-    }
-
-    @Override
-    public int hashCode() {
-        return id;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null) {
-            return false;
-        }
-        if (o.getClass() == getClass()) {
-            return ((SqlVersion) o).id == id;
-        }
-        if (o.getClass() == String.class) {
-            try {
-                SqlVersion v = fromString((String) o);
-                return this.equals(v);
-            } catch (IllegalArgumentException e) {
-                return false;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public int compareTo(SqlVersion o) {
-        return id - o.id;
-    }
-
-    public static int majorMinorId(SqlVersion v) {
-        return v.major * MAJOR_MULTIPLIER + v.minor * MINOR_MULTIPLIER;
-    }
-
-    public int compareToMajorMinor(SqlVersion o) {
-        return majorMinorId(this) - majorMinorId(o);
-    }
-
-    public static boolean hasVersionCompatibility(SqlVersion version) {
-        return version.compareTo(V_7_7_0) >= 0;
-    }
-}
diff --git a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java
deleted file mode 100644
index de8d8bb6ec0ea..0000000000000
--- a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.sql.proto;
-
-import org.elasticsearch.test.ESTestCase;
-
-public class SqlVersionTests extends ESTestCase {
-    public void test123FromString() {
-        SqlVersion ver = SqlVersion.fromString("1.2.3");
-        assertEquals(1, ver.major);
-        assertEquals(2, ver.minor);
-        assertEquals(3, ver.revision);
-        assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id);
-        assertEquals("1.2.3", ver.version);
-    }
-
-    public void test123AlphaFromString() {
-        SqlVersion ver = SqlVersion.fromString("1.2.3-Alpha");
-        assertEquals(1, ver.major);
-        assertEquals(2, ver.minor);
-        assertEquals(3, ver.revision);
-        assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id);
-        assertEquals("1.2.3-Alpha", ver.version);
-    }
-
-    public void test123AlphaSnapshotFromString() {
-        SqlVersion ver = SqlVersion.fromString("1.2.3-Alpha-SNAPSHOT");
-        assertEquals(1, ver.major);
-        assertEquals(2, ver.minor);
-        assertEquals(3, ver.revision);
-        assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id);
-        assertEquals("1.2.3-Alpha-SNAPSHOT", ver.version);
-    }
-
-    public void testVersionsEqual() {
-        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
-        SqlVersion ver2 = SqlVersion.fromString("1.2.3");
-        assertEquals(ver1, ver2);
-    }
-
-    public void testVersionsAndStringEqual() {
-        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
-        String ver2 = "1.2.3";
-        assertEquals(ver1, ver2);
-    }
-
-    public void testVersionsAndStringNotEqual() {
-        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
-        String ver2 = "1.2.4";
-        assertNotEquals(ver1, ver2);
-    }
-
-    public void testVersionsAndInvalidStringNotEqual() {
-        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
-        String ver2 = "invalid";
-        assertNotEquals(ver1, ver2);
-    }
-
-}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
index 671d90fbd54f6..0455039a34e25 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
@@ -236,7 +235,7 @@ public Expression visitPredicated(PredicatedContext ctx) {
                 e = new Like(source, exp, visitPattern(pCtx.pattern()));
                 break;
             case SqlBaseParser.RLIKE:
-                e = new RLike(source, exp, new RLikePattern(string(pCtx.regex)));
+                e = new RLike(source, exp, string(pCtx.regex));
                 break;
             case SqlBaseParser.NULL:
                 // shortcut to avoid double negation later on (since there's no IsNull (missing in ES is a negated exists))
@@ -264,9 +263,6 @@ public LikePattern visitPattern(PatternContext ctx) {
         }
 
         String pattern = string(ctx.value);
-        if (pattern == null) {
-            throw new ParsingException(source(ctx.value), "Pattern must not be [null]");
-        }
         int pos = pattern.indexOf('*');
         if (pos >= 0) {
             throw new ParsingException(source(ctx.value),
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
index 8a3453cda5509..412f5475fbfc2 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
@@ -60,7 +60,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
          */
         String accept = null;
 
-        if (Mode.isDedicatedClient(sqlRequest.requestInfo().mode())
+        if ((Mode.isDriver(sqlRequest.requestInfo().mode()) || sqlRequest.requestInfo().mode() == Mode.CLI)
                 && (sqlRequest.binaryCommunication() == null || sqlRequest.binaryCommunication())) {
             // enforce CBOR response for drivers and CLI (unless instructed differently through the config param)
             accept = XContentType.CBOR.name();
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
index bb5156a47b35b..e958ce1aa830d 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
@@ -36,7 +36,6 @@
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.TO_NULL;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -160,7 +159,7 @@ public static Converter converterFor(DataType from, DataType to) {
         }
         // extend the default converter with DATE and TIME
         if (from == DATE || from == TIME) {
-            if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
+            if (to == KEYWORD || to == TEXT) {
                 return conversionToString(from);
             }
             if (to == LONG) {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
index 616b2de40810a..6c321bea80de8 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
@@ -29,7 +29,6 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -269,7 +268,6 @@ public static boolean isFromDocValuesOnly(DataType dataType) {
                 || dataType == DATE         // because of date formats
                 || dataType == DATETIME
                 || dataType == SCALED_FLOAT // because of scaling_factor
-                || dataType == CONSTANT_KEYWORD
                 || dataType == GEO_POINT
                 || dataType == GEO_SHAPE
                 || dataType == SHAPE;
@@ -335,9 +333,6 @@ public static SQLType sqlType(DataType dataType) {
         if (dataType == TEXT) {
             return JDBCType.VARCHAR;
         }
-        if (dataType == CONSTANT_KEYWORD) {
-            return JDBCType.VARCHAR;
-        }
         if (dataType == DATETIME) {
             return JDBCType.TIMESTAMP;
         }
@@ -461,9 +456,6 @@ public static int defaultPrecision(DataType dataType) {
         if (dataType == TEXT) {
             return 32766;
         }
-        if (dataType == CONSTANT_KEYWORD) {
-            return 15;
-        }
         if (dataType == DATETIME) {
             return 3;
         }
@@ -584,9 +576,6 @@ public static int displaySize(DataType dataType) {
         if (dataType == TEXT) {
             return dataType.size();
         }
-        if (dataType == CONSTANT_KEYWORD) {
-            return 32766;
-        }
         if (dataType == DATETIME) {
             return 29;
         }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java
index f5a834acade80..5f85ff90e344c 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java
@@ -5,14 +5,12 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.xpack.sql.proto.ColumnInfo;
 import org.elasticsearch.xpack.sql.proto.Mode;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertRequestBuilderThrows;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -30,7 +28,7 @@ public void testSqlAction() {
         boolean dataBeforeCount = randomBoolean();
         String columns = dataBeforeCount ? "data, count" : "count, data";
         SqlQueryResponse response = new SqlQueryRequestBuilder(client(), SqlQueryAction.INSTANCE)
-                .query("SELECT " + columns + " FROM test ORDER BY count").mode(Mode.JDBC).version(Version.CURRENT.toString()).get();
+                .query("SELECT " + columns + " FROM test ORDER BY count").mode(Mode.JDBC).get();
         assertThat(response.size(), equalTo(2L));
         assertThat(response.columns(), hasSize(2));
         int dataIndex = dataBeforeCount ? 0 : 1;
@@ -44,18 +42,5 @@ public void testSqlAction() {
         assertEquals("baz", response.rows().get(1).get(dataIndex));
         assertEquals(43L, response.rows().get(1).get(countIndex));
     }
-
-    public void testSqlActionCurrentVersion() {
-        SqlQueryResponse response = new SqlQueryRequestBuilder(client(), SqlQueryAction.INSTANCE)
-            .query("SELECT true").mode(randomFrom(Mode.CLI, Mode.JDBC)).version(Version.CURRENT.toString()).get();
-        assertThat(response.size(), equalTo(1L));
-        assertEquals(true, response.rows().get(0).get(0));
-    }
-
-    public void testSqlActionOutdatedVersion() {
-        SqlQueryRequestBuilder request = new SqlQueryRequestBuilder(client(), SqlQueryAction.INSTANCE)
-            .query("SELECT true").mode(randomFrom(Mode.CLI, Mode.JDBC)).version("1.2.3");
-        assertRequestBuilderThrows(request, org.elasticsearch.action.ActionRequestValidationException.class);
-    }
 }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
index 614cc24ec44aa..b53d1cbdf8133 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
@@ -10,12 +10,10 @@
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
-import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.ql.type.InvalidMappedField;
 import org.elasticsearch.xpack.ql.type.KeywordEsField;
-import org.elasticsearch.xpack.ql.type.TextEsField;
 import org.elasticsearch.xpack.sql.type.SqlDataTypeRegistry;
 
 import java.util.ArrayList;
@@ -28,7 +26,6 @@
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
@@ -271,52 +268,6 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
                 ((InvalidMappedField) esField).errorMessage());
     }
 
-    public void testConstantKeywordTwoIndicesScenario() throws Exception {
-        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
-        addFieldCaps(fieldCaps, "name", "text", true, false);
-        addFieldCaps(fieldCaps, "wheels_count", "constant_keyword", true, true);
-        addFieldCaps(fieldCaps, "motor", "keyword", true, true);
-
-        Map<String, FieldCapabilities> multi = new HashMap<>();
-        multi.put("keyword", new FieldCapabilities("cycle_type", "keyword", true, true, new String[] { "other_cycles" }, null, null,
-                Collections.emptyMap()));
-        multi.put("constant_keyword", new FieldCapabilities("cycle_type", "constant_keyword", true, false, new String[] { "bicycles" },
-                null, null, Collections.emptyMap()));
-        fieldCaps.put("cycle_type", multi);
-
-        String wildcard = "*";
-        IndexResolution resolution = mergedMappings(wildcard, new String[] { "other_cycles", "bicycles" }, fieldCaps);
-
-        assertTrue(resolution.isValid());
-
-        EsIndex esIndex = resolution.get();
-        assertEquals(wildcard, esIndex.name());
-        EsField esField = null;
-        Map<String, EsField> props = esIndex.mapping();
-
-        assertEquals(4, props.size());
-        assertTrue(props.containsKey("cycle_type"));
-        assertTrue(props.containsKey("name"));
-        assertTrue(props.containsKey("wheels_count"));
-        assertTrue(props.containsKey("motor"));
-        
-        esField = props.get("cycle_type");
-        assertTrue(esField instanceof KeywordEsField);
-        assertEquals(KEYWORD, ((KeywordEsField) esField).getDataType());
-        
-        esField = props.get("name");
-        assertTrue(esField instanceof TextEsField);
-        assertEquals(TEXT, ((TextEsField) esField).getDataType());
-        
-        esField = props.get("wheels_count");
-        assertTrue(esField instanceof ConstantKeywordEsField);
-        assertEquals(CONSTANT_KEYWORD, ((ConstantKeywordEsField) esField).getDataType());
-        
-        esField = props.get("motor");
-        assertTrue(esField instanceof KeywordEsField);
-        assertEquals(KEYWORD, ((KeywordEsField) esField).getDataType());
-    }
-
     public void testSeparateSameMappingDifferentIndices() throws Exception {
         Map<String, EsField> oneMapping = loadMapping("mapping-basic.json", true);
         Map<String, EsField> sameMapping = loadMapping("mapping-basic.json", true);
@@ -432,7 +383,7 @@ private static boolean isSearchable(DataType type) {
     }
 
     private static boolean isAggregatable(DataType type) {
-        return type.isNumeric() || type == KEYWORD || type == DATETIME || type == CONSTANT_KEYWORD;
+        return type.isNumeric() || type == KEYWORD || type == DATETIME;
     }
 
     private static class UpdateableFieldCapabilities extends FieldCapabilities {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
index 922b32d199f7f..d4ef29f667ee0 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
@@ -370,7 +369,7 @@ public void testGenericNullableExpression() {
         // comparison
         assertNullLiteral(rule.rule(new GreaterThan(EMPTY, getFieldAttribute(), NULL)));
         // regex
-        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123"))));
+        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, "123")));
     }
 
     public void testNullFoldingOnCast() {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
index 6211f40e0f0e5..ad2d3066e121f 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
@@ -21,19 +21,16 @@
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Sub;
-import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue;
 
 import java.time.Duration;
 import java.time.Period;
 import java.time.temporal.TemporalAmount;
-import java.util.Collections;
 import java.util.Locale;
 
 import static java.lang.String.format;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.LONG;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -545,11 +542,4 @@ public void testCaseWithOperand() {
         assertEquals("WHEN 1 THEN 'one'", ifc.sourceText());
         assertEquals("many", c.elseResult().toString());
     }
-
-    public void testLikePatternWithNullParameterNotAllowed() {
-        ParsingException e = expectThrows(ParsingException.class,
-                () -> parser.createExpression("a LIKE ?",
-                    Collections.singletonList(new SqlTypedParamValue(KEYWORD.typeName(), null))));
-        assertEquals("line 1:9: Pattern must not be [null]", e.getMessage());
-    }
 }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
index e0bf79b830e00..17bc17bbec4b8 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
@@ -46,7 +46,7 @@ public void testSysTypes() {
         Command cmd = sql("SYS TYPES").v1();
 
         List<String> names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", "SCALED_FLOAT",
-                "CONSTANT_KEYWORD", "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME",
+                "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME",
                 "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND",
                 "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND",
                 "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND",
@@ -107,9 +107,7 @@ public void testSysTypesMultipleMatches() {
 
         cmd.execute(session(), wrap(p -> {
             SchemaRowSet r = (SchemaRowSet) p.rowSet();
-            assertEquals(4, r.size());
-            assertEquals("CONSTANT_KEYWORD", r.column(0));
-            assertTrue(r.advanceRow());
+            assertEquals(3, r.size());
             assertEquals("IP", r.column(0));
             assertTrue(r.advanceRow());
             assertEquals("KEYWORD", r.column(0));
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
index 2225ead0dfd31..a70a0d610c36b 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
@@ -547,23 +547,6 @@ private void assertDifferentRLikeAndNotRLikePatterns(String firstPattern, String
         assertEquals("keyword", rqsq.field());
     }
 
-    public void testLikeRLikeAsPainlessScripts() {
-        LogicalPlan p = plan("SELECT count(*), CASE WHEN keyword LIKE '%foo%' THEN 1 WHEN keyword RLIKE '.*bar.*' THEN 2 " +
-                             "ELSE 3 END AS t FROM test GROUP BY t");
-        assertTrue(p instanceof Aggregate);
-        Expression condition = ((Aggregate) p).groupings().get(0);
-        assertFalse(condition.foldable());
-        GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
-        assertNotNull(groupingContext);
-        ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.caseFunction([InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
-                     "doc,params.v0),params.v1),params.v2,InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
-                     "doc,params.v3),params.v4),params.v5,params.v6])",
-                scriptTemplate.toString());
-        assertEquals("[{v=keyword}, {v=^.*foo.*$}, {v=1}, {v=keyword}, {v=.*bar.*}, {v=2}, {v=3}]",
-                scriptTemplate.params().toString());
-    }
-
     public void testTranslateNotExpression_WhereClause_Painless() {
         LogicalPlan p = plan("SELECT * FROM test WHERE NOT(POSITION('x', keyword) = 0)");
         assertTrue(p instanceof Project);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
index c445a84b236ca..b39dd8bcaa897 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
@@ -22,7 +22,6 @@
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -632,7 +631,6 @@ public void testCommonType() {
         assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
         assertEquals(NULL, commonType(NULL, NULL));
         assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
-        assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
         assertEquals(LONG, commonType(TEXT, LONG));
         assertEquals(SHORT, commonType(SHORT, BYTE));
         assertEquals(FLOAT, commonType(BYTE, FLOAT));
@@ -642,11 +640,6 @@ public void testCommonType() {
         // strings
         assertEquals(TEXT, commonType(TEXT, KEYWORD));
         assertEquals(TEXT, commonType(KEYWORD, TEXT));
-        assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
-        assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
-        assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
-        assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
-        assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
 
         // numeric and intervals
         assertEquals(INTERVAL_YEAR_TO_MONTH, commonType(INTERVAL_YEAR_TO_MONTH, LONG));
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
deleted file mode 100644
index ab9171b7bc064..0000000000000
--- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- *
- *  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- *  * or more contributor license agreements. Licensed under the Elastic License;
- *  * you may not use this file except in compliance with the Elastic License.
- *
- */
-
-package org.elasticsearch.xpack.test.rest;
-
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.test.rest.ESRestTestCase;
-
-import java.io.IOException;
-
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.hamcrest.Matchers.matchesRegex;
-
-public class CatIndicesWithSecurityIT extends ESRestTestCase {
-    @Override
-    protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("cat_test_user", new SecureString("cat-test-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    public void testHiddenIndexWithVisibleAlias() throws IOException {
-        // Create the index and alias
-        {
-            final Request createRequest = new Request("PUT", ".index_hidden");
-            createRequest.setJsonEntity(
-                "{\"settings\": {\"index.hidden\": true, \"number_of_replicas\":  0}, \"aliases\": {\"index_allowed\": {}}}");
-            final Response createResponse = adminClient().performRequest(createRequest);
-            assertOK(createResponse);
-            ensureGreen("index_allowed");
-        }
-
-        // And it should be visible by a user that has permissions for only the alias
-        {
-            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
-            final Response catResponse = client().performRequest(catRequest);
-            assertOK(catResponse);
-
-            final String resp = EntityUtils.toString(catResponse.getEntity());
-            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
-        }
-    }
-
-    public void testHiddenIndexWithHiddenAlias() throws IOException {
-        // Create the index and alias
-        {
-            final Request createRequest = new Request("PUT", ".index_hidden");
-            createRequest.setJsonEntity("{\"settings\": {\"index.hidden\": true, \"number_of_replicas\":  0}, "
-                + "\"aliases\": {\"index_allowed\": {\"is_hidden\":  true}}}");
-            final Response createResponse = adminClient().performRequest(createRequest);
-            assertOK(createResponse);
-            ensureGreen("index_allowed");
-        }
-
-        // It should *not* be visible
-        {
-            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
-            final Response catResponse = client().performRequest(catRequest);
-            assertOK(catResponse);
-
-            final String resp = EntityUtils.toString(catResponse.getEntity());
-            assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
-        }
-
-        // But we should be able to see the index if we ask for hidden things
-        {
-            final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
-            final Response catResponse = client().performRequest(catRequest);
-            assertOK(catResponse);
-
-            final String resp = EntityUtils.toString(catResponse.getEntity());
-            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
-        }
-    }
-
-    public void testVisibleIndexWithHiddenAlias() throws IOException {
-        // Create the index and alias
-        {
-            final Request createRequest = new Request("PUT", "visible_index");
-            createRequest.setJsonEntity("{\"settings\": {\"number_of_replicas\":  0}, "
-                + "\"aliases\": {\"index_allowed\": {\"is_hidden\":  true}}}");
-            final Response createResponse = adminClient().performRequest(createRequest);
-            assertOK(createResponse);
-            ensureGreen("index_allowed");
-        }
-
-        // It should *not* be visible, because this user only has access through the *hidden* alias
-        {
-            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
-            final Response catResponse = client().performRequest(catRequest);
-            assertOK(catResponse);
-
-            final String resp = EntityUtils.toString(catResponse.getEntity());
-            assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
-        }
-
-        // But we should be able to see the index if we ask for hidden things
-        {
-            final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
-            final Response catResponse = client().performRequest(catRequest);
-            assertOK(catResponse);
-
-            final String resp = EntityUtils.toString(catResponse.getEntity());
-            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+visible_index.*$"));
-        }
-    }
-}
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
index dd207033554e8..3d5d92c8eb753 100644
--- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.CheckedFunction;
@@ -25,7 +24,6 @@
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
-import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.junit.After;
 import org.junit.Before;
 
@@ -77,7 +75,7 @@ public void setupForTests() throws Exception {
     }
 
     /**
-     * Waits for Machine Learning and Transform templates to be created by the {@link MetaDataUpgrader}
+     * Waits for the Security template and the Machine Learning templates to be created by the {@link MetaDataUpgrader}
      */
     private void waitForTemplates() throws Exception {
         if (installTemplates()) {
@@ -88,10 +86,7 @@ private void waitForTemplates() throws Exception {
                     MlMetaIndex.INDEX_NAME,
                     AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
                     AnomalyDetectorsIndex.jobResultsIndexPrefix(),
-                    AnomalyDetectorsIndex.configIndexName(),
-                    TransformInternalIndexConstants.AUDIT_INDEX,
-                    TransformInternalIndexConstants.LATEST_INDEX_NAME
-                ));
+                    AnomalyDetectorsIndex.configIndexName()));
 
             for (String template : templates) {
                 awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(),
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
deleted file mode 100644
index 4486b90cb9a1d..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-  "async_search.delete":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
-    },
-    "stability":"experimental",
-    "url":{
-      "paths":[
-        {
-          "path":"/_async_search/{id}",
-          "methods":[
-            "DELETE"
-          ],
-          "parts":{
-            "id":{
-              "type":"string",
-              "description":"The async search ID"
-            }
-          }
-        }
-      ]
-    }
-  }
-}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
deleted file mode 100644
index f5ea1424756e1..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-  "async_search.get":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
-    },
-    "stability":"experimental",
-    "url":{
-      "paths":[
-        {
-          "path":"/_async_search/{id}",
-          "methods":[
-            "GET"
-          ],
-          "parts":{
-            "id":{
-              "type":"string",
-              "description":"The async search ID"
-            }
-          }
-        }
-      ]
-    },
-    "params":{
-      "wait_for_completion":{
-        "type":"time",
-        "description":"Specify the time that the request should block waiting for the final response",
-        "default": "1s"
-      },
-      "keep_alive": {
-        "type": "time",
-        "description": "Specify the time interval in which the results (partial or final) for this search will be available",
-        "default": "5d"
-      },
-      "typed_keys":{
-        "type":"boolean",
-        "description":"Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
-      }
-    }
-  }
-}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
deleted file mode 100644
index 033939acf0703..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
+++ /dev/null
@@ -1,225 +0,0 @@
-{
-  "async_search.submit":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
-    },
-    "stability":"experimental",
-    "url":{
-      "paths":[
-        {
-          "path":"/_async_search",
-          "methods":[
-            "POST"
-          ]
-        },
-        {
-          "path":"/{index}/_async_search",
-          "methods":[
-            "POST"
-          ],
-          "parts":{
-            "index":{
-              "type":"list",
-              "description":"A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
-            }
-          }
-        }
-      ]
-    },
-    "params":{
-      "wait_for_completion":{
-        "type":"time",
-        "description":"Specify the time that the request should block waiting for the final response",
-        "default": "1s"
-      },
-      "keep_alive": {
-        "type": "time",
-        "description": "Update the time interval in which the results (partial or final) for this search will be available"
-      },
-      "batched_reduce_size":{
-        "type":"number",
-        "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.",
-        "default":5
-      },
-      "analyzer":{
-        "type":"string",
-        "description":"The analyzer to use for the query string"
-      },
-      "analyze_wildcard":{
-        "type":"boolean",
-        "description":"Specify whether wildcard and prefix queries should be analyzed (default: false)"
-      },
-      "default_operator":{
-        "type":"enum",
-        "options":[
-          "AND",
-          "OR"
-        ],
-        "default":"OR",
-        "description":"The default operator for query string query (AND or OR)"
-      },
-      "df":{
-        "type":"string",
-        "description":"The field to use as default where no field prefix is given in the query string"
-      },
-      "explain":{
-        "type":"boolean",
-        "description":"Specify whether to return detailed information about score computation as part of a hit"
-      },
-      "stored_fields":{
-        "type":"list",
-        "description":"A comma-separated list of stored fields to return as part of a hit"
-      },
-      "docvalue_fields":{
-        "type":"list",
-        "description":"A comma-separated list of fields to return as the docvalue representation of a field for each hit"
-      },
-      "from":{
-        "type":"number",
-        "description":"Starting offset (default: 0)"
-      },
-      "ignore_unavailable":{
-        "type":"boolean",
-        "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)"
-      },
-      "ignore_throttled":{
-        "type":"boolean",
-        "description":"Whether specified concrete, expanded or aliased indices should be ignored when throttled"
-      },
-      "allow_no_indices":{
-        "type":"boolean",
-        "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
-      },
-      "expand_wildcards":{
-        "type":"enum",
-        "options":[
-          "open",
-          "closed",
-          "none",
-          "all"
-        ],
-        "default":"open",
-        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
-      },
-      "lenient":{
-        "type":"boolean",
-        "description":"Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
-      },
-      "preference":{
-        "type":"string",
-        "description":"Specify the node or shard the operation should be performed on (default: random)"
-      },
-      "q":{
-        "type":"string",
-        "description":"Query in the Lucene query string syntax"
-      },
-      "routing":{
-        "type":"list",
-        "description":"A comma-separated list of specific routing values"
-      },
-      "search_type":{
-        "type":"enum",
-        "options":[
-          "query_then_fetch",
-          "dfs_query_then_fetch"
-        ],
-        "description":"Search operation type"
-      },
-      "size":{
-        "type":"number",
-        "description":"Number of hits to return (default: 10)"
-      },
-      "sort":{
-        "type":"list",
-        "description":"A comma-separated list of <field>:<direction> pairs"
-      },
-      "_source":{
-        "type":"list",
-        "description":"True or false to return the _source field or not, or a list of fields to return"
-      },
-      "_source_excludes":{
-        "type":"list",
-        "description":"A list of fields to exclude from the returned _source field"
-      },
-      "_source_includes":{
-        "type":"list",
-        "description":"A list of fields to extract and return from the _source field"
-      },
-      "terminate_after":{
-        "type":"number",
-        "description":"The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early."
-      },
-      "stats":{
-        "type":"list",
-        "description":"Specific 'tag' of the request for logging and statistical purposes"
-      },
-      "suggest_field":{
-        "type":"string",
-        "description":"Specify which field to use for suggestions"
-      },
-      "suggest_mode":{
-        "type":"enum",
-        "options":[
-          "missing",
-          "popular",
-          "always"
-        ],
-        "default":"missing",
-        "description":"Specify suggest mode"
-      },
-      "suggest_size":{
-        "type":"number",
-        "description":"How many suggestions to return in response"
-      },
-      "suggest_text":{
-        "type":"string",
-        "description":"The source text for which the suggestions should be returned"
-      },
-      "timeout":{
-        "type":"time",
-        "description":"Explicit operation timeout"
-      },
-      "track_scores":{
-        "type":"boolean",
-        "description":"Whether to calculate and return scores even if they are not used for sorting"
-      },
-      "track_total_hits":{
-        "type":"boolean",
-        "description":"Indicate if the number of documents that match the query should be tracked"
-      },
-      "allow_partial_search_results":{
-        "type":"boolean",
-        "default":true,
-        "description":"Indicate if an error should be returned if there is a partial search failure or timeout"
-      },
-      "typed_keys":{
-        "type":"boolean",
-        "description":"Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
-      },
-      "version":{
-        "type":"boolean",
-        "description":"Specify whether to return document version as part of a hit"
-      },
-      "seq_no_primary_term":{
-        "type":"boolean",
-        "description":"Specify whether to return sequence number and primary term of the last modification of each hit"
-      },
-      "request_cache":{
-        "type":"boolean",
-        "description":"Specify if request cache should be used for this request or not, defaults to index level setting"
-      },
-      "max_concurrent_shard_requests":{
-        "type":"number",
-        "description":"The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests",
-        "default":5
-      },
-      "clean_on_completion":{
-        "type":"boolean",
-        "description":"Control whether the response should not be stored in the cluster if it completed within the provided [wait_for_completion] time (default: true)"
-      }
-    },
-    "body":{
-      "description":"The search definition using the Query DSL"
-    }
-  }
-}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json
deleted file mode 100644
index 6eaa707a4969f..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
-  "ml.estimate_model_memory":{
-    "documentation":{
-      "url":null
-    },
-    "stability":"stable",
-    "url":{
-      "paths":[
-        {
-          "path":"/_ml/anomaly_detectors/_estimate_model_memory",
-          "methods":[
-            "POST"
-          ]
-        }
-      ]
-    },
-    "params":{},
-    "body":{
-      "description":"The analysis config, plus cardinality estimates for fields it references",
-      "required":true
-    }
-  }
-}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
deleted file mode 100644
index ef457a569ad0e..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
+++ /dev/null
@@ -1,143 +0,0 @@
----
-"Async search":
-  - do:
-      indices.create:
-        index: test-1
-        body:
-          settings:
-            number_of_shards: "2"
-
-  - do:
-      indices.create:
-        index: test-2
-        body:
-          settings:
-            number_of_shards: "1"
-
-  - do:
-      indices.create:
-        index: test-3
-        body:
-          settings:
-            number_of_shards: "3"
-
-  - do:
-      index:
-        index:  test-2
-        body:   { max: 2 }
-
-  - do:
-      index:
-        index:  test-1
-        body:   { max: 1 }
-
-  - do:
-      index:
-        index:  test-3
-        body:   { max: 3 }
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      async_search.submit:
-        index: test-*
-        batched_reduce_size: 2
-        wait_for_completion: 10s
-        body:
-          query:
-            match_all: {}
-          aggs:
-            1:
-              max:
-                field: max
-          sort: max
-
-  - is_false: id
-  - match:  { version:                          6 }
-  - match:  { is_partial:                   false }
-  - length: { response.hits.hits:               3 }
-  - match:  { response.hits.hits.0._source.max: 1 }
-  - match:  { response.aggregations.1.value:  3.0 }
-
-  - do:
-      async_search.submit:
-        index: test-*
-        batched_reduce_size: 2
-        wait_for_completion: 10s
-        clean_on_completion: false
-        body:
-          aggs:
-            1:
-              max:
-                field: max
-          sort: max
-
-  - set:    { id:                              id }
-  - match:  { version:                          6 }
-  - match:  { is_partial:                   false }
-  - length: { response.hits.hits:               3 }
-  - match:  { response.hits.hits.0._source.max: 1 }
-  - match:  { response.aggregations.1.value:  3.0 }
-
-  # test with typed_keys:
-  - do:
-      async_search.submit:
-        index: test-*
-        batched_reduce_size: 2
-        wait_for_completion: 10s
-        clean_on_completion: false
-        typed_keys: true
-        body:
-          aggs:
-            1:
-              max:
-                field: max
-          sort: max
-
-  - set:    { id:                                  id }
-  - match:  { version:                              6 }
-  - match:  { is_partial:                       false }
-  - length: { response.hits.hits:                   3 }
-  - match:  { response.hits.hits.0._source.max:     1 }
-  - match:  { response.aggregations.max#1.value:  3.0 }
-
-  - do:
-      async_search.get:
-        id: "$id"
-
-  - match:  { version:                            6 }
-  - match:  { is_partial:                     false }
-  - length: { response.hits.hits:                 3 }
-  - match:  { response.hits.hits.0._source.max:   1 }
-  - match:  { response.aggregations.1.value:    3.0 }
-
-  # test with typed_keys:
-  - do:
-      async_search.get:
-        id: "$id"
-        typed_keys: true
-
-  - match:  { version:                              6 }
-  - match:  { is_partial:                       false }
-  - length: { response.hits.hits:                   3 }
-  - match:  { response.hits.hits.0._source.max:     1 }
-  - match:  { response.aggregations.max#1.value:  3.0 }
-
-  - do:
-      async_search.delete:
-        id: "$id"
-
-  - match: { acknowledged:   true }
-
-  - do:
-      catch: missing
-      async_search.get:
-        id: "$id"
-
-  - do:
-      catch: missing
-      async_search.delete:
-        id: "$id"
-
-
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
index a5a99b30391ea..df394a85f0995 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -1333,12 +1333,12 @@ setup:
           }
 
 ---
-"Test put regression given max_trees is zero":
+"Test put regression given maximum_number_trees is zero":
 
   - do:
-      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "regression-max-trees-is-zero"
+        id: "regression-maximum-number-trees-is-zero"
         body: >
           {
             "source": {
@@ -1350,18 +1350,18 @@ setup:
             "analysis": {
               "regression": {
                 "dependent_variable": "foo",
-                "max_trees": 0
+                "maximum_number_trees": 0
               }
             }
           }
 
 ---
-"Test put regression given max_trees is greater than 2k":
+"Test put regression given maximum_number_trees is greater than 2k":
 
   - do:
-      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "regression-max-trees-greater-than-2k"
+        id: "regression-maximum-number-trees-greater-than-2k"
         body: >
           {
             "source": {
@@ -1373,7 +1373,7 @@ setup:
             "analysis": {
               "regression": {
                 "dependent_variable": "foo",
-                "max_trees": 2001
+                "maximum_number_trees": 2001
               }
             }
           }
@@ -1490,7 +1490,7 @@ setup:
                 "lambda": 3.14,
                 "gamma": 0.42,
                 "eta": 0.5,
-                "max_trees": 400,
+                "maximum_number_trees": 400,
                 "feature_bag_fraction": 0.3,
                 "training_percent": 60.3,
                 "randomize_seed": 42
@@ -1506,7 +1506,7 @@ setup:
       "lambda": 3.14,
       "gamma": 0.42,
       "eta": 0.5,
-      "max_trees": 400,
+      "maximum_number_trees": 400,
       "feature_bag_fraction": 0.3,
       "prediction_field_name": "foo_prediction",
       "training_percent": 60.3,
@@ -1629,12 +1629,12 @@ setup:
           }
 
 ---
-"Test put classification given max_trees is zero":
+"Test put classification given maximum_number_trees is zero":
 
   - do:
-      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "classification-max-trees-is-zero"
+        id: "classification-maximum-number-trees-is-zero"
         body: >
           {
             "source": {
@@ -1646,18 +1646,18 @@ setup:
             "analysis": {
               "classification": {
                 "dependent_variable": "foo",
-                "max_trees": 0
+                "maximum_number_trees": 0
               }
             }
           }
 
 ---
-"Test put classification given max_trees is greater than 2k":
+"Test put classification given maximum_number_trees is greater than 2k":
 
   - do:
-      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "classification-max-trees-greater-than-2k"
+        id: "classification-maximum-number-trees-greater-than-2k"
         body: >
           {
             "source": {
@@ -1669,7 +1669,7 @@ setup:
             "analysis": {
               "classification": {
                 "dependent_variable": "foo",
-                "max_trees": 2001
+                "maximum_number_trees": 2001
               }
             }
           }
@@ -1832,9 +1832,8 @@ setup:
                 "lambda": 3.14,
                 "gamma": 0.42,
                 "eta": 0.5,
-                "max_trees": 400,
+                "maximum_number_trees": 400,
                 "feature_bag_fraction": 0.3,
-                "class_assignment_objective": "maximize_accuracy",
                 "training_percent": 60.3,
                 "randomize_seed": 24
               }
@@ -1849,56 +1848,11 @@ setup:
       "lambda": 3.14,
       "gamma": 0.42,
       "eta": 0.5,
-      "max_trees": 400,
+      "maximum_number_trees": 400,
       "feature_bag_fraction": 0.3,
       "prediction_field_name": "foo_prediction",
       "training_percent": 60.3,
       "randomize_seed": 24,
-      "class_assignment_objective": "maximize_accuracy",
-      "num_top_classes": 2
-    }
-  }}
-  - is_true: create_time
-  - is_true: version
-
----
-"Test put classification given deprecated maximum_number_trees":
-
-  - skip:
-      features: allowed_warnings
-
-  - do:
-      allowed_warnings:
-        - 'Deprecated field [maximum_number_trees] used, expected [max_trees] instead'
-      ml.put_data_frame_analytics:
-        id: "classification-with-maximum-number-trees"
-        body: >
-          {
-            "source": {
-              "index": "index-source"
-            },
-            "dest": {
-              "index": "index-dest"
-            },
-            "analysis": {
-              "classification": {
-                "dependent_variable": "foo",
-                "maximum_number_trees": 400,
-                "randomize_seed": 24
-              }
-            }
-          }
-  - match: { id: "classification-with-maximum-number-trees" }
-  - match: { source.index: ["index-source"] }
-  - match: { dest.index: "index-dest" }
-  - match: { analysis: {
-    "classification":{
-      "dependent_variable": "foo",
-      "max_trees": 400,
-      "prediction_field_name": "foo_prediction",
-      "training_percent": 100.0,
-      "randomize_seed": 24,
-      "class_assignment_objective": "maximize_minimum_recall",
       "num_top_classes": 2
     }
   }}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
deleted file mode 100644
index 8253f12dda70e..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
+++ /dev/null
@@ -1,172 +0,0 @@
----
-"Test by field":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline"}]
-            },
-            "overall_cardinality": {
-              "airline": 50000
-            }
-          }
-  - match: { model_memory_estimate: "3135mb" }
-
----
-"Test by field also influencer":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline"}],
-              "influencers": [ "airline" ]
-            },
-            "overall_cardinality": {
-              "airline": 50000
-            },
-            "max_bucket_cardinality": {
-              "airline": 500
-            }
-          }
-  - match: { model_memory_estimate: "3135mb" }
-
----
-"Test by field with independent influencer":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline"}],
-              "influencers": [ "country" ]
-            },
-            "overall_cardinality": {
-              "airline": 50000
-            },
-            "max_bucket_cardinality": {
-              "country": 500
-            }
-          }
-  - match: { model_memory_estimate: "3140mb" }
-
----
-"Test partition field":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "partition_field_name": "airline"}]
-            },
-            "overall_cardinality": {
-              "airline": 50000
-            }
-          }
-  - match: { model_memory_estimate: "3135mb" }
-
----
-"Test partition field also influencer":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "partition_field_name": "airline"}],
-              "influencers": [ "airline" ]
-            },
-            "overall_cardinality": {
-              "airline": 50000
-            },
-            "max_bucket_cardinality": {
-              "airline": 500
-            }
-          }
-  - match: { model_memory_estimate: "3135mb" }
-
----
-"Test partition field with independent influencer":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "partition_field_name": "airline"}],
-              "influencers": [ "country" ]
-            },
-            "overall_cardinality": {
-              "airline": 50000
-            },
-            "max_bucket_cardinality": {
-              "country": 500
-            }
-          }
-  - match: { model_memory_estimate: "3140mb" }
-
----
-"Test by and partition field":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline", "partition_field_name": "country"}]
-            },
-            "overall_cardinality": {
-              "airline": 4000,
-              "country": 600
-            }
-          }
-  - match: { model_memory_estimate: "150010mb" }
-
----
-"Test by and partition fields also influencers":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline", "partition_field_name": "country"}],
-              "influencers": [ "airline", "country" ]
-            },
-            "overall_cardinality": {
-              "airline": 4000,
-              "country": 600
-            },
-            "max_bucket_cardinality": {
-              "airline": 60,
-              "country": 40
-            }
-          }
-  - match: { model_memory_estimate: "150010mb" }
-
----
-"Test by and partition fields with independent influencer":
-  - do:
-      ml.estimate_model_memory:
-        body: >
-          {
-            "analysis_config": {
-              "bucket_span": "1h",
-              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline", "partition_field_name": "country"}],
-              "influencers": [ "src_ip" ]
-            },
-            "overall_cardinality": {
-              "airline": 4000,
-              "country": 600
-            },
-            "max_bucket_cardinality": {
-              "src_ip": 500
-            }
-          }
-  - match: { model_memory_estimate: "150015mb" }
-
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
index 6774ae8b455cd..28ef49182361f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
@@ -111,7 +111,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       headers:
@@ -464,7 +464,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       headers:
@@ -514,7 +514,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       ml.delete_job:
@@ -607,7 +607,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       ml.delete_job:
@@ -685,7 +685,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       headers:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
index 8c1f1e581de5b..cb5ccd90e1311 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
@@ -313,7 +313,7 @@ setup:
                   "model_id" : "a-regression-model-0",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_map": {}
+                  "field_mappings": {}
                 }
               }
             ]
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
deleted file mode 100644
index 9c043eee3cd2c..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
+++ /dev/null
@@ -1,114 +0,0 @@
-# These tests are not part of the ml-with-security tests
-# These are more like ingest processor tests than actual ML focused tests.
-setup:
-  - skip:
-      features: headers
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_trained_model:
-        model_id: a-perfect-regression-model
-        body: >
-          {
-            "description": "empty model for tests",
-            "tags": ["regression", "tag1"],
-            "input": {"field_names": ["field1", "field2"]},
-            "definition": {
-               "preprocessors": [],
-               "trained_model": {
-                  "tree": {
-                     "feature_names": ["field1", "field2"],
-                     "tree_structure": [
-                        {"node_index": 0, "leaf_value": 42}
-                     ],
-                     "target_type": "regression"
-                  }
-               }
-            }
-          }
-
----
-"Test create and delete pipeline with inference processor":
-  - do:
-      ingest.put_pipeline:
-        id: "regression-model-pipeline"
-        body:  >
-          {
-            "processors": [
-              {
-                "inference" : {
-                  "model_id" : "a-perfect-regression-model",
-                  "inference_config": {"regression": {}},
-                  "target_field": "regression_field",
-                  "field_map": {}
-                }
-              }
-            ]
-          }
-  - match: { acknowledged: true }
-  - do:
-      ingest.delete_pipeline:
-        id: "regression-model-pipeline"
----
-"Test create processor with missing mandatory fields":
-  - do:
-      catch: /\[model_id\] required property is missing/
-      ingest.put_pipeline:
-        id: "regression-model-pipeline"
-        body:  >
-          {
-            "processors": [
-              {
-                "inference" : {
-                  "inference_config": {"regression": {}},
-                  "target_field": "regression_field",
-                  "field_map": {}
-                }
-              }
-            ]
-          }
-  - do:
-      catch: /\[inference_config\] required property is missing/
-      ingest.put_pipeline:
-        id: "regression-model-pipeline"
-        body:  >
-          {
-            "processors": [
-              {
-                "inference" : {
-                  "model_id" : "a-perfect-regression-model",
-                  "target_field": "regression_field",
-                  "field_map": {}
-                }
-              }
-            ]
-          }
----
-"Test create processor with deprecated fields":
-  - skip:
-      features:
-       - "warnings"
-       - "allowed_warnings"
-  - do:
-      warnings:
-        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
-      ingest.put_pipeline:
-        id: "regression-model-pipeline"
-        body:  >
-          {
-            "processors": [
-              {
-                "inference" : {
-                  "model_id" : "a-perfect-regression-model",
-                  "inference_config": {"regression": {}},
-                  "field_mappings": {}
-                }
-              }
-            ]
-          }
-
-  - do:
-      allowed_warnings:
-        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
-      ingest.delete_pipeline:
-        id: "regression-model-pipeline"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
index 40d0f6ba01ab7..5143a690b018a 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
@@ -66,7 +66,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_map": {}
+                  "field_mappings": {}
                 }
               }
             ]
@@ -84,7 +84,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_map": {}
+                  "field_mappings": {}
                 }
               }
             ]
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
index 445ab11b04288..a5242b5062e1f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
@@ -471,7 +471,7 @@
 
   - do:
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       catch: /Invalid update value for analysis_limits[:] model_memory_limit cannot be decreased below current usage; current usage \[9mb\], update had \[5mb\]/
@@ -928,7 +928,7 @@
 
   - do:
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       catch: /status_exception/
@@ -956,7 +956,7 @@
 
   - do:
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       catch: /status_exception/
@@ -994,7 +994,7 @@
 
   - do:
       indices.refresh:
-        expand_wildcards: all
+        index: ["*",".ml-*"]
 
   - do:
       catch: /status_exception/
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
index ab6cfac9515e5..81077b1e69a8c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
@@ -142,7 +142,7 @@
         id: "start_given_empty_dest_index"
 
 ---
-"Test start classification analysis when the dependent variable cardinality is too low":
+"Test start classification analysis when the dependent variable cardinality is too low or too high":
   - do:
       indices.create:
         index: index-with-dep-var-with-too-high-card
@@ -179,3 +179,22 @@
       catch: /Field \[keyword_field\] must have at least \[2\] distinct values but there were \[1\]/
       ml.start_data_frame_analytics:
         id: "classification-cardinality-limits"
+
+  - do:
+      index:
+        index: index-with-dep-var-with-too-high-card
+        body: { numeric_field: 2.0, keyword_field: "class_b" }
+
+  - do:
+      index:
+        index: index-with-dep-var-with-too-high-card
+        body: { numeric_field: 3.0, keyword_field: "class_c" }
+
+  - do:
+      indices.refresh:
+        index: index-with-dep-var-with-too-high-card
+
+  - do:
+      catch: /Field \[keyword_field\] must have at most \[2\] distinct values but there were at least \[3\]/
+      ml.start_data_frame_analytics:
+        id: "classification-cardinality-limits"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
deleted file mode 100644
index c67a79c8218da..0000000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
+++ /dev/null
@@ -1,218 +0,0 @@
-setup:
-  - skip:
-      features: headers
-      version: " - 7.9.99"
-      reason: "wildcard fields were added from 8.0"
-
-  - do:
-      indices.create:
-        index: test-index
-        body:
-          settings:
-            number_of_replicas: 0
-          mappings:
-            properties:
-              my_wildcard:
-                 type: wildcard
-  - do:
-      index:
-        index: test-index
-        id: 1
-        body:
-          my_wildcard: hello world
-  - do:
-      index:
-        index: test-index
-        id: 2
-        body:
-          my_wildcard: goodbye world
-
-  - do:
-      indices.refresh: {}
-
----
-"Short prefix query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "hel*" }
-
-
-  - match: {hits.total.value: 1}
-
----
-"Long prefix query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "hello wor*" }
-
-
-  - match: {hits.total.value: 1}
-
----
-"Short unrooted query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "*ello*" }
-
-
-  - match: {hits.total.value: 1}
-
----
-"Long unrooted query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "*ello worl*" }
-
-
-  - match: {hits.total.value: 1}
-
----
-"Short suffix query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "*ld" }
-
-
-  - match: {hits.total.value: 2}
-
----
-"Long suffix query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "*ello world" }
-
-
-  - match: {hits.total.value: 1}
-
----
-"No wildcard wildcard query":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "hello world" }
-
-
-  - match: {hits.total.value: 1}
-
----
-"Term query on wildcard field":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            term:
-              my_wildcard: "hello world"
-
-
-  - match: {hits.total.value: 1}
-
----
-"Terms query on wildcard field":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            terms:
-              my_wildcard: ["hello world", "does not exist"]
-
-
-  - match: {hits.total.value: 1}
-
----
-"Prefix query on wildcard field":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            prefix:
-              my_wildcard:
-                value: "hell*"
-
-
-  - match: {hits.total.value: 1}
-
----
-"Sequence fail":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "*world*hello*" }
-
-
-  - match: {hits.total.value: 0}
-
----
-"Aggs work":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          query:
-            wildcard:
-              my_wildcard: {value: "*world*" }
-          aggs:
-            top_vals:
-              terms: {field: "my_wildcard" }
-
-
-  - match: {hits.total.value: 2}
-  - length: { aggregations.top_vals.buckets: 2 }
-
----
-"Sort works":
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          sort: [ { "my_wildcard": "desc" } ]
-
-  - match: { hits.total.value: 2 }
-  - length: { hits.hits: 2 }
-  - match: { hits.hits.0._id: "1" }
-  - match: { hits.hits.1._id: "2" }
-
-  - do:
-      search:
-        body:
-          track_total_hits: true
-          sort: [ { "my_wildcard": "asc" } ]
-
-  - match: { hits.total.value: 2 }
-  - length: { hits.hits: 2 }
-  - match: { hits.hits.0._id: "2" }
-  - match: { hits.hits.1._id: "1" }
-
-
diff --git a/x-pack/plugin/src/test/resources/roles.yml b/x-pack/plugin/src/test/resources/roles.yml
deleted file mode 100644
index 5f8677860d079..0000000000000
--- a/x-pack/plugin/src/test/resources/roles.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-# Required for CatIndicesWithSecurityIT
-cat_test_role:
-  cluster:
-    - monitor
-  indices:
-    - names: [ "index_allowed" ]
-      privileges: [ "read", "write", "monitor" ]
diff --git a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
index 7dd064f7dbc91..96dde81bf8a45 100644
--- a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
+++ b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
@@ -14,11 +14,11 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -54,10 +54,7 @@ public final void startWatcher() throws Exception {
                     throw new AssertionError("waiting until starting state reached started state");
                 case "started":
                     int watcherCount = (int) response.evaluate("stats.0.watch_count");
-                    if (watcherCount > 0) {
-                        logger.info("expected 0 active watches, but got [{}], deleting watcher indices again", watcherCount);
-                        deleteWatcherIndices();
-                    }
+                    assertThat(watcherCount, equalTo(0));
                     // all good here, we are done
                     break;
                 default:
@@ -91,10 +88,7 @@ public final void stopWatcher() throws Exception {
                     throw new AssertionError("unknown state[" + state + "]");
             }
         }, 60, TimeUnit.SECONDS);
-        deleteWatcherIndices();
-    }
 
-    private static void deleteWatcherIndices() throws IOException {
         Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
         deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
         adminClient().performRequest(deleteWatchesIndexRequest);
diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle
deleted file mode 100644
index 22e29aa0832f2..0000000000000
--- a/x-pack/plugin/wildcard/build.gradle
+++ /dev/null
@@ -1,18 +0,0 @@
-evaluationDependsOn(xpackModule('core'))
-
-apply plugin: 'elasticsearch.esplugin'
-
-esplugin {
-  name 'wildcard'
-  description 'A plugin for a keyword field type with efficient wildcard search'
-  classname 'org.elasticsearch.xpack.wildcard.Wildcard'
-  extendedPlugins = ['x-pack-core']
-}
-archivesBaseName = 'x-pack-wildcard'
-
-dependencies {
-  compileOnly project(path: xpackModule('core'), configuration: 'default')
-  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-}
-
-integTest.enabled = false
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
deleted file mode 100644
index 3749dc2622c8b..0000000000000
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.wildcard;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.plugins.MapperPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.xpack.wildcard.mapper.WildcardFieldMapper;
-
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class Wildcard extends Plugin implements MapperPlugin {
-
-
-    public Wildcard(Settings settings) {
-    }
-
-    @Override
-    public Map<String, Mapper.TypeParser> getMappers() {
-        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
-        mappers.put(WildcardFieldMapper.CONTENT_TYPE, new WildcardFieldMapper.TypeParser());
-        return Collections.unmodifiableMap(mappers);
-    }
-}
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
deleted file mode 100644
index 648fbc7e0cdc3..0000000000000
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.wildcard.mapper;
-
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * Query that runs an Automaton across all binary doc values. 
- * Expensive to run so normally used in conjunction with more selective query clauses.
- */
-public class AutomatonQueryOnBinaryDv extends Query {
-
-    private final String field;
-    private final String matchPattern;
-    private final Automaton automaton;
-
-    public AutomatonQueryOnBinaryDv(String field, String matchPattern, Automaton automaton) {
-        this.field = field;
-        this.matchPattern = matchPattern;
-        this.automaton = automaton;
-    }
-
-    @Override
-    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
-                
-        ByteRunAutomaton bytesMatcher = new ByteRunAutomaton(automaton);
-        
-        return new ConstantScoreWeight(this, boost) {
-
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                ByteArrayDataInput badi = new ByteArrayDataInput();
-                final BinaryDocValues values = DocValues.getBinary(context.reader(), field);               
-                TwoPhaseIterator twoPhase = new TwoPhaseIterator(values) {
-                    @Override
-                    public boolean matches() throws IOException {
-                        BytesRef arrayOfValues = values.binaryValue();
-                        badi.reset(arrayOfValues.bytes);
-                        badi.setPosition(arrayOfValues.offset);
-                        
-                        int size = badi.readVInt();
-                        for (int i=0; i< size; i++) {
-                            int valLength = badi.readVInt();
-                            if (bytesMatcher.run(arrayOfValues.bytes, badi.getPosition(), valLength)) {
-                                return true;
-                            }
-                            badi.skipBytes(valLength);
-                        }
-                        return false;
-                    }
-
-                    @Override
-                    public float matchCost() {
-                        // TODO: how can we compute this?
-                        return 1000f;
-                    }
-                };
-                return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
-            }
-
-            @Override
-            public boolean isCacheable(LeafReaderContext ctx) {
-                return true;
-            }
-        };
-    }
-    @Override
-    public String toString(String field) {
-        return field+":"+matchPattern;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        AutomatonQueryOnBinaryDv other = (AutomatonQueryOnBinaryDv) obj;
-        return Objects.equals(field, other.field)  && Objects.equals(matchPattern, other.matchPattern);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(field, matchPattern);
-    }
-
-}
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
deleted file mode 100644
index e489d8a35bb9f..0000000000000
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ /dev/null
@@ -1,575 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-
-package org.elasticsearch.xpack.wildcard.mapper;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ngram.NGramTokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.MultiTermQuery.RewriteMethod;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.util.automaton.Automaton;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.lucene.BytesRefs;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalyzerScope;
-import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexFieldDataCache;
-import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
-import org.elasticsearch.index.mapper.BinaryFieldMapper.CustomBinaryDocValuesField;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.similarity.SimilarityProvider;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.index.mapper.TypeParsers.parseField;
-
-/**
- * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching
- */
-public class WildcardFieldMapper extends FieldMapper {
-
-    public static final String CONTENT_TYPE = "wildcard";
-    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; 
-    public static final int NGRAM_SIZE = 3;        
-    static final NamedAnalyzer WILDCARD_ANALYZER = new NamedAnalyzer("_wildcard", AnalyzerScope.GLOBAL, new Analyzer() {
-        @Override
-        public TokenStreamComponents createComponents(String fieldName) {
-            Tokenizer tokenizer = new NGramTokenizer(NGRAM_SIZE, NGRAM_SIZE);
-            return new TokenStreamComponents(tokenizer);
-        }
-    });    
-
-    public static class Defaults {
-        public static final MappedFieldType FIELD_TYPE = new WildcardFieldType();
-
-        static {
-            FIELD_TYPE.setTokenized(false);
-            FIELD_TYPE.setIndexAnalyzer(WILDCARD_ANALYZER);
-            FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
-            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
-            FIELD_TYPE.setStoreTermVectorOffsets(false);
-            FIELD_TYPE.setOmitNorms(true);
-            FIELD_TYPE.freeze();
-        }
-        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;        
-    }
-
-    public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
-        protected int ignoreAbove = Defaults.IGNORE_ABOVE;
-
-        public Builder(String name) {
-            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
-            builder = this;
-        }
-
-        @Override
-        public Builder docValues(boolean docValues) {
-            if (docValues == false) {
-                throw new MapperParsingException("The field [" + name + "] cannot have doc values = false");                
-            }
-            return this;
-        }
-
-        @Override
-        public Builder indexOptions(IndexOptions indexOptions) {
-            if (indexOptions != IndexOptions.DOCS) {
-                throw new MapperParsingException("The field [" + name + "] cannot have indexOptions = " + indexOptions);
-            }
-            return this;
-        }
-
-        @Override
-        public Builder store(boolean store) {
-            if (store) {
-                throw new MapperParsingException("The field [" + name + "] cannot have store = true");                
-            }
-            return this;
-        }
-
-        @Override
-        public Builder similarity(SimilarityProvider similarity) {
-            throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");                
-        }
-
-        @Override
-        public Builder index(boolean index) {
-            if (index == false) {
-                throw new MapperParsingException("The field [" + name + "] cannot have index = false");                
-            }
-            return this;
-        }
-        
-        public Builder ignoreAbove(int ignoreAbove) {
-            if (ignoreAbove < 0) {
-                throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
-            }
-            this.ignoreAbove = ignoreAbove;
-            return this;
-        }        
-          
-        
-        @Override
-        protected void setupFieldType(BuilderContext context) {
-            super.setupFieldType(context);
-            fieldType().setHasDocValues(true);
-            fieldType().setTokenized(false);
-            fieldType().setIndexOptions(IndexOptions.DOCS);                
-        }
-
-        @Override
-        public WildcardFieldType fieldType() {
-            return (WildcardFieldType) super.fieldType();
-        }
-
-        @Override
-        public WildcardFieldMapper build(BuilderContext context) {
-            setupFieldType(context);            
-            return new WildcardFieldMapper(
-                    name, fieldType, defaultFieldType, ignoreAbove, 
-                    context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
-        }
-    }
-
-    public static class TypeParser implements Mapper.TypeParser {
-        @Override
-        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
-                throws MapperParsingException {
-            WildcardFieldMapper.Builder builder = new WildcardFieldMapper.Builder(name);
-            parseField(builder, name, node, parserContext);
-            
-            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
-                Map.Entry<String, Object> entry = iterator.next();
-                String propName = entry.getKey();
-                Object propNode = entry.getValue();
-                if (propName.equals("ignore_above")) {
-                    builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
-                    iterator.remove();
-                }
-            }            
-            
-            return builder;
-        }
-    }
-    
-     public static final char TOKEN_START_OR_END_CHAR = 0;    
-    
-     public static final class WildcardFieldType extends MappedFieldType {
-
-        public WildcardFieldType() {            
-            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
-            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);            
-        }
-
-        protected WildcardFieldType(WildcardFieldType ref) {
-            super(ref);
-        }
-
-        public WildcardFieldType clone() {
-            WildcardFieldType result = new WildcardFieldType(this);
-            return result;
-        }
-        
-                
-        // Holds parsed information about the wildcard pattern
-        static class PatternStructure {
-            boolean openStart, openEnd, hasSymbols;            
-            int lastGap =0;
-            int wildcardCharCount, wildcardStringCount;
-            String[] fragments;
-            Integer []  precedingGapSizes;
-            final String pattern;
-            
-            @SuppressWarnings("fallthrough") // Intentionally uses fallthrough mirroring implementation in Lucene's WildcardQuery
-            PatternStructure (String wildcardText) {
-                this.pattern = wildcardText;
-                ArrayList<String> fragmentList = new ArrayList<>();
-                ArrayList<Integer> precedingGapSizeList = new ArrayList<>();
-                StringBuilder sb = new StringBuilder();               
-                for (int i = 0; i < wildcardText.length();) {
-                    final int c = wildcardText.codePointAt(i);
-                    int length = Character.charCount(c);
-                    switch (c) {
-                    case WildcardQuery.WILDCARD_STRING:
-                        if (i == 0) {
-                            openStart = true;
-                        }
-                        openEnd = true;                        
-                        hasSymbols = true;
-                        wildcardStringCount++;
-                        
-                        if (sb.length() > 0) {
-                            precedingGapSizeList.add(lastGap);
-                            fragmentList.add(sb.toString());
-                            sb = new StringBuilder();
-                        }
-                        lastGap = Integer.MAX_VALUE;
-                        break;
-                    case WildcardQuery.WILDCARD_CHAR:
-                        if (i == 0) {
-                            openStart = true;
-                        }
-                        hasSymbols = true;
-                        wildcardCharCount++;
-                        openEnd = true;
-                        if (sb.length() > 0) {
-                            precedingGapSizeList.add(lastGap);
-                            fragmentList.add(sb.toString());
-                            sb = new StringBuilder();       
-                            lastGap = 0;
-                        }
-                        
-                        if (lastGap != Integer.MAX_VALUE) {
-                            lastGap++;
-                        }
-                        break;
-                    case WildcardQuery.WILDCARD_ESCAPE:
-                        // add the next codepoint instead, if it exists
-                        if (i + length < wildcardText.length()) {
-                            final int nextChar = wildcardText.codePointAt(i + length);
-                            length += Character.charCount(nextChar);
-                            sb.append(Character.toChars(nextChar));
-                            openEnd = false;
-                            break;
-                        } // else fallthru, lenient parsing with a trailing \
-                    default:
-                        openEnd = false;
-                        sb.append(Character.toChars(c));
-                    }
-                    i += length;
-                }
-                if (sb.length() > 0) {
-                    precedingGapSizeList.add(lastGap);
-                    fragmentList.add(sb.toString());
-                    lastGap = 0;
-                }
-                fragments = fragmentList.toArray(new String[0]);
-                precedingGapSizes = precedingGapSizeList.toArray(new Integer[0]);
-                
-            }
-            
-            public boolean needsVerification() {
-                // Return true if term queries are not enough evidence
-                if (fragments.length == 1 && wildcardCharCount == 0) {
-                    // The one case where we don't need verification is when 
-                    // we have a single fragment and no ? characters
-                    return false;
-                }
-                return true;
-            }
-            
-            // Returns number of positions for last gap (Integer.MAX means unlimited gap)
-            public int getPrecedingGapSize(int fragmentNum) {
-                return precedingGapSizes[fragmentNum];
-            }
-
-            public boolean isMatchAll() {
-                return fragments.length == 0 && wildcardStringCount >0 && wildcardCharCount ==0;
-            }
-
-            @Override
-            public int hashCode() {
-                return pattern.hashCode();
-            }
-
-            @Override
-            public boolean equals(Object obj) {
-                PatternStructure other = (PatternStructure) obj;
-                return pattern.equals(other.pattern);
-            }
-            
-            
-        }
-        
-
-        @Override
-        public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
-            PatternStructure patternStructure = new PatternStructure(wildcardPattern);            
-            ArrayList<String> tokens = new ArrayList<>();
-
-            for (int i = 0; i < patternStructure.fragments.length; i++) {
-                String fragment = patternStructure.fragments[i];
-                int fLength = fragment.length();
-                if (fLength == 0) {
-                    continue;
-                }
-                
-                // Add any start/end of string character
-                if (i == 0 && patternStructure.openStart == false) {
-                    // Start-of-string anchored (is not a leading wildcard)
-                    fragment = TOKEN_START_OR_END_CHAR + fragment;
-                }
-                if (patternStructure.openEnd == false && i == patternStructure.fragments.length - 1) {
-                    // End-of-string anchored (is not a trailing wildcard)
-                    fragment = fragment + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
-                }
-                if (fragment.codePointCount(0, fragment.length()) <= NGRAM_SIZE) {
-                    tokens.add(fragment);
-                } else {
-                    // Break fragment into multiple Ngrams                
-                    TokenStream tokenizer = WILDCARD_ANALYZER.tokenStream(name(), fragment);
-                    CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class);
-                    String lastUnusedToken = null;
-                    try {
-                        tokenizer.reset();
-                        boolean takeThis = true;
-                        // minimise number of terms searched - eg for "12345" and 3grams we only need terms
-                        // `123` and `345` - no need to search for 234. We take every other ngram.
-                        while (tokenizer.incrementToken()) {
-                            String tokenValue = termAtt.toString();
-                            if (takeThis) {
-                                tokens.add(tokenValue);
-                            } else {
-                                lastUnusedToken = tokenValue;                                
-                            }
-                            // alternate
-                            takeThis = !takeThis;
-                        }
-                        if (lastUnusedToken != null) {
-                            // given `cake` and 3 grams the loop above would output only `cak` and we need to add trailing
-                            // `ake` to complete the logic.
-                            tokens.add(lastUnusedToken);
-                        }
-                        tokenizer.end();
-                        tokenizer.close();
-                    } catch (IOException ioe) {
-                        throw new ElasticsearchParseException("Error parsing wildcard query pattern fragment [" + fragment + "]");
-                    }
-                }
-            }
-
-            if (patternStructure.isMatchAll()) {
-                return new MatchAllDocsQuery();
-            } 
-            BooleanQuery approximation = createApproximationQuery(tokens);
-            if (approximation.clauses().size() > 1 || patternStructure.needsVerification()) {
-                BooleanQuery.Builder verifyingBuilder = new BooleanQuery.Builder();
-                verifyingBuilder.add(new BooleanClause(approximation, Occur.MUST));
-                Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), wildcardPattern));
-                verifyingBuilder.add(new BooleanClause(new AutomatonQueryOnBinaryDv(name(), wildcardPattern, automaton), Occur.MUST));
-                return verifyingBuilder.build();
-            }
-            return approximation;
-        }                
-
-        private BooleanQuery createApproximationQuery(ArrayList<String> tokens) {
-            BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();
-            if (tokens.size() <= MAX_CLAUSES_IN_APPROXIMATION_QUERY) {
-                for (String token : tokens) {
-                    addClause(token, bqBuilder);
-                }
-                return bqBuilder.build();
-            }
-            // Thin out the number of clauses using a selection spread evenly across the range
-            float step = (float) (tokens.size() - 1) / (float) (MAX_CLAUSES_IN_APPROXIMATION_QUERY - 1); // set step size
-            for (int i = 0; i < MAX_CLAUSES_IN_APPROXIMATION_QUERY; i++) {
-                addClause(tokens.get(Math.round(step * i)), bqBuilder); // add each element of a position which is a multiple of step
-            }
-            // TODO we can be smarter about pruning here. e.g.
-            // * Avoid wildcard queries if there are sufficient numbers of other terms that are full 3grams that are cheaper term queries
-            // * We can select terms on their scarcity rather than even spreads across the search string.
-            
-            return bqBuilder.build();
-        }
-
-        private void addClause(String token, BooleanQuery.Builder bqBuilder) {
-            assert token.codePointCount(0, token.length()) <= NGRAM_SIZE;
-            if (token.codePointCount(0, token.length()) == NGRAM_SIZE) {
-                TermQuery tq = new TermQuery(new Term(name(), token));
-                bqBuilder.add(new BooleanClause(tq, Occur.MUST));
-            } else {
-                WildcardQuery wq = new WildcardQuery(new Term(name(), token + "*"));
-                wq.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
-                bqBuilder.add(new BooleanClause(wq, Occur.MUST));
-            }
-
-        }
-
-        @Override
-        public String typeName() {
-            return CONTENT_TYPE;
-        }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
-
-        @Override
-        public Query termQuery(Object value, QueryShardContext context) {
-            return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, context);
-        }
-        
-        @Override
-        public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-            return wildcardQuery(value + "*", method, context);
-        }        
-
-        @Override
-        public Query termsQuery(List<?> values, QueryShardContext context) {
-            BooleanQuery.Builder bq = new BooleanQuery.Builder();
-            for (Object value : values) {
-                bq.add(termQuery(value, context), Occur.SHOULD);
-            }
-            return new ConstantScoreQuery(bq.build());
-        }     
-                
-        @Override
-        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
-            failIfNoDocValues();
-            return new IndexFieldData.Builder() {
-
-                @Override
-                public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
-                        CircuitBreakerService breakerService, MapperService mapperService) {
-                    return new WildcardBytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
-                }};
-        }        
-    }
-     
-    static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
-
-        WildcardBytesBinaryDVIndexFieldData(Index index, String fieldName) {
-            super(index, fieldName);
-        }
-
-        @Override
-        public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
-            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue,
-                    sortMode, nested);
-            return new SortField(getFieldName(), source, reverse);
-        }
-    
-    }
-
-    private int ignoreAbove;
-
-    private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
-                int ignoreAbove, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
-        super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
-        this.ignoreAbove = ignoreAbove;
-        assert fieldType.indexOptions() == IndexOptions.DOCS;
-        
-        ngramFieldType = fieldType.clone();
-        ngramFieldType.setTokenized(true);                    
-        ngramFieldType.freeze();
-    }
-
-    /** Values that have more chars than the return value of this method will
-     *  be skipped at parsing time. */
-    // pkg-private for testing
-    int ignoreAbove() {
-        return ignoreAbove;
-    }    
-    
-    @Override
-    protected WildcardFieldMapper clone() {
-        return (WildcardFieldMapper) super.clone();
-    }
-
-    @Override
-    public WildcardFieldType fieldType() {
-        return (WildcardFieldType) super.fieldType();
-    }
-
-    @Override
-    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
-        super.doXContentBody(builder, includeDefaults, params);
-        if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
-            builder.field("ignore_above", ignoreAbove);
-        }
-    }
-    
-    @Override
-    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
-        final String value;
-        if (context.externalValueSet()) {
-            value = context.externalValue().toString();
-        } else {
-            XContentParser parser = context.parser();
-            if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
-                value = fieldType().nullValueAsString();
-            } else {
-                value =  parser.textOrNull();
-            }
-        }
-        ParseContext.Document parseDoc = context.doc();
-        
-        createFields(value, parseDoc, fields);        
-    }   
-    
-    // For internal use by Lucene only - used to define ngram index
-    final MappedFieldType ngramFieldType;
-    
-    void createFields(String value, Document parseDoc, List<IndexableField>fields) {
-        if (value == null || value.length() > ignoreAbove) {
-            return;
-        }
-        String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
-        Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
-        fields.add(ngramField);
-        
-        CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name());
-        if (dvField == null) {
-            dvField = new CustomBinaryDocValuesField(fieldType().name(), value.getBytes(StandardCharsets.UTF_8));
-            parseDoc.addWithKey(fieldType().name(), dvField);
-        } else {
-            dvField.add(value.getBytes(StandardCharsets.UTF_8));
-        }        
-    }
-
-    @Override
-    protected String contentType() {
-        return CONTENT_TYPE;
-    }
-    
-
-    @Override
-    protected void doMerge(Mapper mergeWith) {
-        super.doMerge(mergeWith);
-        this.ignoreAbove = ((WildcardFieldMapper) mergeWith).ignoreAbove;
-    }    
-}
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
deleted file mode 100644
index f8c8ddc7f5eef..0000000000000
--- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.wildcard.mapper;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldDataCache;
-import org.elasticsearch.index.mapper.ContentPath;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.xpack.wildcard.mapper.WildcardFieldMapper.Builder;
-import org.junit.Before;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.function.BiFunction;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class WildcardFieldMapperTests extends ESTestCase {
-
-    private static final String KEYWORD_FIELD_NAME = "keyword_field";
-    private static final String WILDCARD_FIELD_NAME = "wildcard_field";
-    static final int MAX_FIELD_LENGTH = 100;
-    static WildcardFieldMapper wildcardFieldType;
-    static KeywordFieldMapper keywordFieldType;
-
-    @Override
-    @Before
-    public void setUp() throws Exception {
-        Builder builder = new WildcardFieldMapper.Builder(WILDCARD_FIELD_NAME);
-        builder.ignoreAbove(MAX_FIELD_LENGTH);        
-        wildcardFieldType = builder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));
-        
-        
-        org.elasticsearch.index.mapper.KeywordFieldMapper.Builder kwBuilder = new KeywordFieldMapper.Builder(KEYWORD_FIELD_NAME);
-        keywordFieldType = kwBuilder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));        
-        super.setUp();
-    }
-    
-    public void testIllegalDocValuesArgument() {
-        Builder ft = new WildcardFieldMapper.Builder("test");
-        MapperParsingException e = expectThrows(MapperParsingException.class,
-                () -> ft.docValues(false));
-        assertEquals("The field [test] cannot have doc values = false", e.getMessage());
-    }        
-
-    public void testIllegalIndexedArgument() {
-        Builder ft = new WildcardFieldMapper.Builder("test");
-        MapperParsingException e = expectThrows(MapperParsingException.class,
-                () -> ft.index(false));
-        assertEquals("The field [test] cannot have index = false", e.getMessage());
-    }
-    
-    public void testTooBigKeywordField() throws IOException {
-        Directory dir = newDirectory();
-        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
-        iwc.setMergePolicy(newTieredMergePolicy(random()));
-        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-        
-        // Create a string that is too large and will not be indexed
-        String docContent = randomABString(MAX_FIELD_LENGTH + 1);
-        Document doc = new Document();
-        ParseContext.Document parseDoc = new ParseContext.Document();                        
-        addFields(parseDoc, doc, docContent);
-        indexDoc(parseDoc, doc, iw);        
-
-        iw.forceMerge(1);
-        DirectoryReader reader = iw.getReader();
-        IndexSearcher searcher = newSearcher(reader);
-        iw.close();
-
-        Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery("*a*", null, null);
-        TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
-        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
-                
-        reader.close();
-        dir.close();        
-    }
-    
-    //Test long query strings don't cause exceptions
-    public void testTooBigQueryField() throws IOException {
-        Directory dir = newDirectory();
-        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
-        iwc.setMergePolicy(newTieredMergePolicy(random()));
-        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-        
-        // Create a string that is too large and will not be indexed
-        String docContent = randomABString(10);
-        Document doc = new Document();
-        ParseContext.Document parseDoc = new ParseContext.Document();                        
-        addFields(parseDoc, doc, docContent);
-        indexDoc(parseDoc, doc, iw);        
-
-        iw.forceMerge(1);
-        DirectoryReader reader = iw.getReader();
-        IndexSearcher searcher = newSearcher(reader);
-        iw.close();
-
-        String queryString = randomABString((BooleanQuery.getMaxClauseCount() * 2) + 1);
-        Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(queryString, null, null);
-        TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
-        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
-                
-        reader.close();
-        dir.close();        
-    }    
-    
-    
-    public void testSearchResultsVersusKeywordField() throws IOException {
-        Directory dir = newDirectory();
-        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
-        iwc.setMergePolicy(newTieredMergePolicy(random()));
-        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
-        int numDocs = 100;
-        HashSet<String> values = new HashSet<>();
-        for (int i = 0; i < numDocs; i++) {
-            Document doc = new Document();
-            ParseContext.Document parseDoc = new ParseContext.Document();                        
-            String docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
-            if (values.contains(docContent) == false) {                
-                addFields(parseDoc, doc, docContent);
-                values.add(docContent);
-            }
-            // Occasionally add a multi-value field
-            if (randomBoolean()) {
-                docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
-                if (values.contains(docContent) == false) {                
-                    addFields(parseDoc, doc, docContent);
-                    values.add(docContent);
-                }                
-            }
-            indexDoc(parseDoc, doc, iw);
-            
-        }
-
-        iw.forceMerge(1);
-        DirectoryReader reader = iw.getReader();
-        IndexSearcher searcher = newSearcher(reader);
-        iw.close();
-
-        int numSearches = 100;
-        for (int i = 0; i < numSearches; i++) {
-            String randomWildcardPattern = getRandomWildcardPattern();
-
-            Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(randomWildcardPattern, null, null);
-            TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, values.size() + 1, Sort.INDEXORDER);
-
-            Query keywordFieldQuery = new WildcardQuery(new Term(KEYWORD_FIELD_NAME, randomWildcardPattern));
-            TopDocs kwTopDocs = searcher.search(keywordFieldQuery, values.size() + 1, Sort.INDEXORDER);
-
-            assertThat(kwTopDocs.totalHits.value, equalTo(wildcardFieldTopDocs.totalHits.value));
-
-            HashSet<Integer> expectedDocs = new HashSet<>();
-            for (ScoreDoc topDoc : kwTopDocs.scoreDocs) {
-                expectedDocs.add(topDoc.doc);
-            }
-            for (ScoreDoc wcTopDoc : wildcardFieldTopDocs.scoreDocs) {
-                assertTrue(expectedDocs.remove(wcTopDoc.doc));
-            }
-            assertThat(expectedDocs.size(), equalTo(0));
-        }
-        
-        
-        //Test keyword and wildcard sort operations are also equivalent
-        QueryShardContext shardContextMock = createMockShardContext();
-        
-        FieldSortBuilder wildcardSortBuilder = new FieldSortBuilder(WILDCARD_FIELD_NAME);
-        SortField wildcardSortField = wildcardSortBuilder.build(shardContextMock).field;        
-        ScoreDoc[] wildcardHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(wildcardSortField)).scoreDocs;
-
-        FieldSortBuilder keywordSortBuilder = new FieldSortBuilder(KEYWORD_FIELD_NAME);
-        SortField keywordSortField = keywordSortBuilder.build(shardContextMock).field;        
-        ScoreDoc[] keywordHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(keywordSortField)).scoreDocs;
-        
-        assertThat(wildcardHits.length, equalTo(keywordHits.length));
-        for (int i = 0; i < wildcardHits.length; i++) {
-            assertThat(wildcardHits[i].doc, equalTo(keywordHits[i].doc));            
-        }
-        
-        reader.close();
-        dir.close();
-    }
-
-    
-    
-    protected MappedFieldType provideMappedFieldType(String name) {
-        if (name.equals(WILDCARD_FIELD_NAME)) {
-            return wildcardFieldType.fieldType();            
-        } else {
-            return keywordFieldType.fieldType();
-        }
-    }    
-    
-    protected final QueryShardContext createMockShardContext() {
-        Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index,
-            Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build());
-        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class));
-        BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup = (fieldType, fieldIndexName) -> {
-            IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName);
-            return builder.build(idxSettings, fieldType, new IndexFieldDataCache.None(), null, null);
-        };
-        return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup,
-                null, null, null, xContentRegistry(), null, null, null,
-                () -> randomNonNegativeLong(), null, null, () -> true) {
-
-            @Override
-            public MappedFieldType fieldMapper(String name) {
-                return provideMappedFieldType(name);
-            }
-        };
-    }       
-    
-    private void addFields(ParseContext.Document parseDoc, Document doc, String docContent) throws IOException {
-        ArrayList<IndexableField> fields = new ArrayList<>();
-        wildcardFieldType.createFields(docContent, parseDoc, fields);
-
-        for (IndexableField indexableField : fields) {
-            doc.add(indexableField);
-        }
-        // Add keyword fields too
-        doc.add(new SortedSetDocValuesField(KEYWORD_FIELD_NAME, new BytesRef(docContent)));        
-        doc.add(new StringField(KEYWORD_FIELD_NAME, docContent, Field.Store.YES));
-    }
-
-    private void indexDoc(ParseContext.Document parseDoc, Document doc, RandomIndexWriter iw) throws IOException {
-        IndexableField field = parseDoc.getByKey(wildcardFieldType.name());
-        if (field != null) {
-            doc.add(field);
-        }
-        iw.addDocument(doc);
-    }
-
-    protected IndexSettings createIndexSettings() {
-        return new IndexSettings(
-                IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
-                        .numberOfShards(1).numberOfReplicas(0).creationDate(System.currentTimeMillis()).build(),
-                Settings.EMPTY);
-    }
-
-
-    static String randomABString(int minLength) {
-        StringBuilder sb = new StringBuilder();
-        while (sb.length() < minLength) {
-            if (randomBoolean()) {
-                sb.append("a");
-            } else {
-                sb.append("b");
-            }
-        }
-        return sb.toString();
-    }
-
-    private void randomSyntaxChar(StringBuilder sb) {
-        switch (randomInt(3)) {
-        case 0:
-            sb.append(WildcardQuery.WILDCARD_CHAR);
-            break;
-        case 1:
-            sb.append(WildcardQuery.WILDCARD_STRING);
-            break;
-        case 2:
-            sb.append(WildcardQuery.WILDCARD_ESCAPE);
-            sb.append(WildcardQuery.WILDCARD_STRING);
-            break;
-        case 3:
-            sb.append(WildcardQuery.WILDCARD_ESCAPE);
-            sb.append(WildcardQuery.WILDCARD_CHAR);
-            break;
-        }
-    }
-
-    private String getRandomWildcardPattern() {
-        StringBuilder sb = new StringBuilder();
-        int numFragments = 1 + randomInt(4);
-        if (randomInt(10) == 1) {
-            randomSyntaxChar(sb);
-        }
-        for (int i = 0; i < numFragments; i++) {
-            if (i > 0) {
-                randomSyntaxChar(sb);
-            }
-            sb.append(randomABString(1 + randomInt(6)));
-        }
-        if (randomInt(10) == 1) {
-            randomSyntaxChar(sb);
-        }
-        return sb.toString();
-    }
-}
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java
deleted file mode 100644
index 0d090e4a25058..0000000000000
--- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-
-package org.elasticsearch.xpack.wildcard.mapper;
-
-import org.elasticsearch.index.mapper.FieldTypeTestCase;
-import org.elasticsearch.index.mapper.MappedFieldType;
-
-public class WildcardFieldTypeTests extends FieldTypeTestCase {
-
-    @Override
-    protected MappedFieldType createDefaultFieldType() {
-        return new WildcardFieldMapper.WildcardFieldType();
-    }  
-}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
index d6651382e4e4d..2dc76febf8dc4 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
@@ -77,16 +77,7 @@ public void testMlConfigIndexMappingsAfterMigration() throws Exception {
             // trigger .ml-config index mappings update
             createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID);
             // assert that the mappings are updated
-            Map<String, Object> configIndexMappings = getConfigIndexMappings();
-
-            // Remove renamed fields
-            if (getOldClusterVersion().before(Version.V_8_0_0)) {
-                configIndexMappings = XContentMapValues.filter(expectedConfigIndexMappings, null, new String[] {
-                    "analysis.properties.*.properties.maximum_number_trees" // This was renamed to max_trees
-                });
-            }
-
-            assertThat(configIndexMappings, equalTo(expectedConfigIndexMappings));
+            assertThat(getConfigIndexMappings(), equalTo(expectedConfigIndexMappings));
         }
     }
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index be388da19be07..af304afc57db6 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
-
 ---
 "Get old outlier_detection job":
 
@@ -97,31 +92,6 @@ setup:
   - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
   - match: { data_frame_analytics.0.state: "stopped" }
 
----
-"Get old classification job":
-
-  - do:
-      ml.get_data_frame_analytics:
-        id: "old_cluster_classification_job"
-  - match: { count: 1 }
-  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
-  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
-  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
-  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
-  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
-  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
-  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
-
----
-"Get old classification job stats":
-
-  - do:
-      ml.get_data_frame_analytics_stats:
-        id: "old_cluster_classification_job"
-  - match: { count: 1 }
-  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
-  - match: { data_frame_analytics.0.state: "stopped" }
-
 ---
 "Put an outlier_detection job on the mixed cluster":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index 10ab98a738e1c..fe160bba15f23 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,7 +1,4 @@
 setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
 
   - do:
       index:
@@ -22,16 +19,6 @@ setup:
             "user": "Kimchy"
           }
 
-  - do:
-      index:
-        index: bwc_ml_classification_job_source
-        body: >
-          {
-            "numeric_field_1": 1.0,
-            "foo": "a",
-            "user": "Kimchy"
-          }
-
   - do:
       indices.refresh:
         index: bwc_ml_*
@@ -77,26 +64,3 @@ setup:
             }
           }
   - match: { id: "old_cluster_regression_job" }
-
----
-"Put classification job on the old cluster":
-
-  - do:
-      ml.put_data_frame_analytics:
-        id: "old_cluster_classification_job"
-        body: >
-          {
-            "source": {
-              "index": "bwc_ml_classification_job_source",
-              "query": {"term" : { "user.keyword" : "Kimchy" }}
-            },
-            "dest": {
-              "index": "old_cluster_classification_job_results"
-            },
-            "analysis": {
-              "classification":{
-                "dependent_variable": "foo"
-              }
-            }
-          }
-  - match: { id: "old_cluster_classification_job" }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index aa4ef717263f9..14438883f0da1 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
-
 ---
 "Get old cluster outlier_detection job":
 
@@ -57,32 +52,6 @@ setup:
   - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
   - match: { data_frame_analytics.0.state: "stopped" }
 
----
-"Get old classification job":
-
-  - do:
-      ml.get_data_frame_analytics:
-        id: "old_cluster_classification_job"
-  - match: { count: 1 }
-  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
-  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
-  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
-  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
-  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
-  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
-  - match: { data_frame_analytics.0.analysis.classification.class_assignment_objective: "maximize_minimum_recall" }
-  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
-
----
-"Get old classification job stats":
-
-  - do:
-      ml.get_data_frame_analytics_stats:
-        id: "old_cluster_classification_job"
-  - match: { count: 1 }
-  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
-  - match: { data_frame_analytics.0.state: "stopped" }
-
 ---
 "Get mixed cluster outlier_detection job":
   - skip:

From 00fe62dbf8cdcd25d592ba971b96049ce511de34 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 16 Mar 2020 16:02:26 +0100
Subject: [PATCH 19/51] Revert "Revert "Merge branch 'master' into
 compat/type-index-get""

This reverts commit a0fce895fa6388c05c668c8d56ca3cc93ab24650.
---
 .ci/matrix-runtime-javas.yml                  |   1 -
 CONTRIBUTING.md                               |   8 +-
 build.gradle                                  |   3 +-
 .../gradle/DistributionDownloadPlugin.java    |  23 +-
 .../gradle/JdkDownloadPlugin.java             |  33 +-
 buildSrc/version.properties                   |   4 +-
 .../client/analytics/ParsedTopMetrics.java    |  12 +-
 .../client/cluster/ProxyModeInfo.java         |   6 +-
 .../client/cluster/RemoteConnectionInfo.java  |   6 +-
 .../client/ml/dataframe/Classification.java   |  81 +-
 .../client/ml/dataframe/Regression.java       |  30 +-
 .../ml/inference/TrainedModelConfig.java      |  27 +-
 .../ml/inference/TrainedModelDefinition.java  |   8 +-
 .../trainedmodel/ensemble/Ensemble.java       |   6 +-
 .../java/org/elasticsearch/client/EqlIT.java  | 133 ++-
 .../client/MachineLearningIT.java             |   8 +-
 .../client/analytics/AnalyticsAggsIT.java     |  55 +-
 .../MlClientDocumentationIT.java              |  10 +-
 .../ml/dataframe/ClassificationTests.java     |   3 +-
 .../client/ml/dataframe/RegressionTests.java  |   2 +-
 .../ml/inference/TrainedModelConfigTests.java |   7 +-
 distribution/docker/docker-compose.yml        |   2 +-
 docs/Versions.asciidoc                        |   4 +-
 .../ml/put-data-frame-analytics.asciidoc      |   3 +-
 docs/plugins/integrations.asciidoc            |   4 +-
 docs/python/index.asciidoc                    |   4 +-
 .../bucket/geohashgrid-aggregation.asciidoc   |   2 +-
 .../metrics/top-metrics-aggregation.asciidoc  |  35 +-
 .../flatten-graph-tokenfilter.asciidoc        |   2 -
 .../word-delimiter-graph-tokenfilter.asciidoc | 551 +++++++++--
 .../word-delimiter-tokenfilter.asciidoc       | 421 +++++++--
 .../simplepattern-tokenizer.asciidoc          |   2 -
 .../simplepatternsplit-tokenizer.asciidoc     |   2 -
 docs/reference/cluster/remote-info.asciidoc   |  36 +-
 docs/reference/cluster/stats.asciidoc         | 299 +++++-
 docs/reference/eql/search.asciidoc            |   9 +
 docs/reference/glossary.asciidoc              | 192 +++-
 .../reference/ilm/policy-definitions.asciidoc |   2 +-
 .../images/analysis/token-graph-basic.svg     |  45 +
 .../images/analysis/token-graph-wd.svg        |  52 ++
 .../images/analysis/token-graph-wdg.svg       |  53 ++
 .../apis/enrich/put-enrich-policy.asciidoc    |  15 +-
 docs/reference/ingest/enrich.asciidoc         |  20 +-
 .../ingest/processors/inference.asciidoc      |   4 +-
 docs/reference/mapping/types.asciidoc         |   4 +-
 .../mapping/types/histogram.asciidoc          |   1 -
 .../mapping/types/search-as-you-type.asciidoc |  15 +-
 .../reference/mapping/types/wildcard.asciidoc |  53 ++
 .../migration/migrate_8_0/snapshots.asciidoc  |   7 +
 .../apis/put-dfanalytics.asciidoc             |  12 +-
 docs/reference/ml/ml-shared.asciidoc          |  28 +-
 .../modules/cross-cluster-search.asciidoc     |  37 +-
 .../modules/remote-clusters.asciidoc          | 156 +++-
 docs/reference/redirects.asciidoc             |  10 +
 .../search/request/stored-fields.asciidoc     |   7 +-
 .../sql/language/data-types.asciidoc          |  31 +-
 .../apis/get-transform-stats.asciidoc         |   4 +
 .../common/xcontent/AbstractObjectParser.java |  25 +
 .../xcontent/ConstructingObjectParser.java    |  84 +-
 .../common/xcontent/ObjectParser.java         |  28 +-
 .../common/xcontent/ObjectParserTests.java    |  73 +-
 modules/ingest-geoip/build.gradle             |   5 +-
 .../jackson-databind-2.8.11.4.jar.sha1        |   1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |   1 +
 ...ssions-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...ssions-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 .../mustache/RestSearchTemplateAction.java    |   1 -
 .../TransportSearchTemplateAction.java        |  21 +
 .../test/lang_mustache/30_search_template.yml |  13 +
 .../50_multi_search_template.yml              |   8 +
 .../org/elasticsearch/painless/Operation.java |  58 +-
 .../painless/node/AExpression.java            | 102 ++-
 .../painless/node/AStatement.java             | 131 +--
 .../painless/node/AStoreable.java             |  19 +-
 .../painless/node/EAssignment.java            | 198 ++--
 .../elasticsearch/painless/node/EBinary.java  | 450 ++-------
 .../elasticsearch/painless/node/EBool.java    |  21 +-
 .../elasticsearch/painless/node/EBoolean.java |  13 +-
 .../painless/node/ECallLocal.java             |  26 +-
 .../painless/node/ECapturingFunctionRef.java  |  17 +-
 .../elasticsearch/painless/node/EComp.java    | 243 +----
 .../painless/node/EConditional.java           |  52 +-
 .../painless/node/EConstant.java              |  32 +-
 .../elasticsearch/painless/node/EDecimal.java |  15 +-
 .../elasticsearch/painless/node/EElvis.java   |  43 +-
 .../painless/node/EExplicit.java              |  18 +-
 .../painless/node/EFunctionRef.java           |  18 +-
 .../painless/node/EInstanceof.java            |  25 +-
 .../elasticsearch/painless/node/ELambda.java  |  34 +-
 .../painless/node/EListInit.java              |  28 +-
 .../elasticsearch/painless/node/EMapInit.java |  35 +-
 .../painless/node/ENewArray.java              |  20 +-
 .../painless/node/ENewArrayFunctionRef.java   |  17 +-
 .../elasticsearch/painless/node/ENewObj.java  |  30 +-
 .../elasticsearch/painless/node/ENull.java    |  21 +-
 .../elasticsearch/painless/node/ENumeric.java |  27 +-
 .../elasticsearch/painless/node/ERegex.java   |  12 +-
 .../elasticsearch/painless/node/EStatic.java  |  13 +-
 .../elasticsearch/painless/node/EString.java  |  13 +-
 .../elasticsearch/painless/node/EUnary.java   | 111 +--
 .../painless/node/EVariable.java              |  24 +-
 .../elasticsearch/painless/node/PBrace.java   |  55 +-
 .../painless/node/PCallInvoke.java            |  34 +-
 .../elasticsearch/painless/node/PField.java   |  68 +-
 .../painless/node/PSubArrayLength.java        |  13 +-
 .../painless/node/PSubBrace.java              |  16 +-
 .../painless/node/PSubCallInvoke.java         |  20 +-
 .../painless/node/PSubDefArray.java           |  17 +-
 .../painless/node/PSubDefCall.java            |  22 +-
 .../painless/node/PSubDefField.java           |  13 +-
 .../painless/node/PSubField.java              |  13 +-
 .../painless/node/PSubListShortcut.java       |  18 +-
 .../painless/node/PSubMapShortcut.java        |  18 +-
 .../painless/node/PSubNullSafeCallInvoke.java |  15 +-
 .../painless/node/PSubNullSafeField.java      |  20 +-
 .../painless/node/PSubShortcut.java           |  13 +-
 .../elasticsearch/painless/node/SBlock.java   |  36 +-
 .../elasticsearch/painless/node/SBreak.java   |  17 +-
 .../elasticsearch/painless/node/SCatch.java   |  32 +-
 .../painless/node/SContinue.java              |  17 +-
 .../painless/node/SDeclBlock.java             |  11 +-
 .../painless/node/SDeclaration.java           |  13 +-
 .../org/elasticsearch/painless/node/SDo.java  |  29 +-
 .../elasticsearch/painless/node/SEach.java    |  32 +-
 .../painless/node/SExpression.java            |  30 +-
 .../org/elasticsearch/painless/node/SFor.java |  52 +-
 .../painless/node/SFunction.java              |   9 +-
 .../org/elasticsearch/painless/node/SIf.java  |  27 +-
 .../elasticsearch/painless/node/SIfElse.java  |  48 +-
 .../elasticsearch/painless/node/SReturn.java  |  22 +-
 .../painless/node/SSubEachArray.java          |  11 +-
 .../painless/node/SSubEachIterable.java       |  13 +-
 .../elasticsearch/painless/node/SThrow.java   |  20 +-
 .../org/elasticsearch/painless/node/STry.java |  49 +-
 .../elasticsearch/painless/node/SWhile.java   |  31 +-
 .../percolator/QueryAnalyzerTests.java        |  13 +-
 ...rs-icu-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...rs-icu-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...romoji-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...romoji-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...s-nori-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...s-nori-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...onetic-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...onetic-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...martcn-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...martcn-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...tempel-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...tempel-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...ologik-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...ologik-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 .../jackson-databind-2.8.11.4.jar.sha1        |   1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |   1 +
 .../jackson-databind-2.8.11.4.jar.sha1        |   1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |   1 +
 .../index/engine/EvilInternalEngineTests.java |   3 +-
 .../test/multi_cluster/20_info.yml            |   6 +-
 .../rest-api-spec/api/cat.aliases.json        |  12 +
 .../rest-api-spec/api/cat.indices.json        |  12 +
 .../rest-api-spec/api/cluster.health.json     |   1 +
 .../rest-api-spec/api/cluster.state.json      |   1 +
 .../resources/rest-api-spec/api/count.json    |   1 +
 .../rest-api-spec/api/delete_by_query.json    |   1 +
 .../rest-api-spec/api/field_caps.json         |   1 +
 .../api/indices.clear_cache.json              |   1 +
 .../rest-api-spec/api/indices.close.json      |   1 +
 .../rest-api-spec/api/indices.delete.json     |   1 +
 .../rest-api-spec/api/indices.exists.json     |   1 +
 .../api/indices.exists_alias.json             |   1 +
 .../api/indices.exists_type.json              |   1 +
 .../rest-api-spec/api/indices.flush.json      |   1 +
 .../rest-api-spec/api/indices.forcemerge.json |   1 +
 .../rest-api-spec/api/indices.get.json        |   1 +
 .../rest-api-spec/api/indices.get_alias.json  |   3 +-
 .../api/indices.get_field_mapping.json        |   1 +
 .../api/indices.get_mapping.json              |   1 +
 .../api/indices.get_settings.json             |   1 +
 .../api/indices.get_upgrade.json              |   1 +
 .../rest-api-spec/api/indices.open.json       |   1 +
 .../api/indices.put_mapping.json              |   1 +
 .../api/indices.put_settings.json             |   1 +
 .../rest-api-spec/api/indices.refresh.json    |   1 +
 .../rest-api-spec/api/indices.segments.json   |   1 +
 .../api/indices.shard_stores.json             |   1 +
 .../rest-api-spec/api/indices.stats.json      |   1 +
 .../rest-api-spec/api/indices.upgrade.json    |   1 +
 .../api/indices.validate_query.json           |   1 +
 .../rest-api-spec/api/rank_eval.json          |   1 +
 .../resources/rest-api-spec/api/search.json   |   1 +
 .../rest-api-spec/api/search_shards.json      |   1 +
 .../rest-api-spec/api/search_template.json    |   1 +
 .../rest-api-spec/api/update_by_query.json    |   1 +
 .../test/cat.aliases/40_hidden.yml            | 150 +++
 .../test/cat.indices/20_hidden.yml            | 243 +++++
 .../test/search.aggregation/230_composite.yml | 137 +++
 .../test/search.aggregation/320_missing.yml   |  98 ++
 .../search.highlight/40_keyword_ignore.yml    |  64 ++
 .../test/search/320_disallow_queries.yml      | 185 +---
 ...common-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...common-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...codecs-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...codecs-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...e-core-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...e-core-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...ouping-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...ouping-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...ighter-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...ighter-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...e-join-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...e-join-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...memory-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...memory-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...e-misc-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...e-misc-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...ueries-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...ueries-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...parser-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...parser-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...andbox-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...andbox-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...extras-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...extras-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...tial3d-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...tial3d-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 ...uggest-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...uggest-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 .../org/apache/lucene/queries/XIntervals.java | 861 ------------------
 .../cluster/node/info/NodesInfoRequest.java   |   2 +-
 .../cluster/node/stats/NodesStatsRequest.java | 200 ++--
 .../restore/RestoreSnapshotRequest.java       |  98 +-
 .../RestoreSnapshotRequestBuilder.java        |  54 --
 .../indices/forcemerge/ForceMergeRequest.java |  30 +
 .../action/fieldcaps/FieldCapabilities.java   |   4 +-
 .../ingest/SimulateExecutionService.java      |   7 +
 .../ingest/SimulatePipelineRequest.java       |   9 +-
 .../search/AbstractSearchAsyncAction.java     |  47 +-
 .../search/CanMatchPreFilterSearchPhase.java  |  34 +-
 .../action/search/ClearScrollController.java  |   2 +-
 .../action/search/DfsQueryPhase.java          |  12 +-
 .../action/search/FetchSearchPhase.java       |  37 +-
 .../action/search/ScrollIdForNode.java        |  13 +-
 .../SearchDfsQueryThenFetchAsyncAction.java   |  24 +-
 .../action/search/SearchPhaseContext.java     |   7 +-
 .../action/search/SearchPhaseController.java  |  13 +-
 .../action/search/SearchProgressListener.java |  51 +-
 .../SearchQueryThenFetchAsyncAction.java      |  28 +-
 .../action/search/SearchRequest.java          |   7 +-
 .../action/search/SearchRequestBuilder.java   |   2 +-
 .../action/search/SearchResponse.java         |   4 +
 .../search/SearchScrollAsyncAction.java       |   9 +-
 ...SearchScrollQueryThenFetchAsyncAction.java |   4 +-
 .../action/search/SearchShard.java            |   8 +-
 .../action/search/SearchShardTask.java        |   1 -
 .../action/search/SearchTask.java             |   4 +-
 .../action/search/SearchTransportService.java |  30 +-
 .../action/search/TransportSearchAction.java  |  12 +-
 .../action/search/TransportSearchHelper.java  |  30 +-
 .../elasticsearch/client/node/NodeClient.java |  38 -
 .../elasticsearch/cluster/ClusterModule.java  |   5 +
 .../cluster/NodeConnectionsService.java       |  38 +-
 .../cluster/coordination/Coordinator.java     |   4 +-
 .../cluster/coordination/JoinHelper.java      |   6 +-
 .../cluster/coordination/JoinRequest.java     |  46 +-
 .../cluster/metadata/ComponentTemplate.java   | 300 ++++++
 .../metadata/ComponentTemplateMetadata.java   | 169 ++++
 .../cluster/metadata/MetaData.java            |  35 +
 .../cluster/node/DiscoveryNodes.java          |  19 +-
 .../command/AllocationCommands.java           |   3 -
 .../SpanBooleanQueryRewriteWithMaxClause.java |   6 +-
 .../elasticsearch/common/util/BitArray.java   |  32 +-
 .../gateway/GatewayAllocator.java             |   1 +
 .../elasticsearch/index/engine/Engine.java    |  11 +-
 .../index/engine/InternalEngine.java          |  30 +-
 .../index/engine/NoOpEngine.java              |  19 +-
 .../index/engine/ReadOnlyEngine.java          |  37 +-
 .../plain/BinaryDVIndexFieldData.java         |  17 +-
 .../index/mapper/DateFieldMapper.java         |  19 +
 .../index/mapper/KeywordFieldMapper.java      |   4 +-
 .../index/mapper/ParseContext.java            |  18 -
 .../index/mapper/StringFieldType.java         |  44 +-
 .../index/mapper/TextFieldMapper.java         |   7 +-
 .../index/mapper/TypeFieldMapper.java         |  61 +-
 .../index/query/IntervalsSourceProvider.java  |   8 +-
 .../elasticsearch/index/shard/IndexShard.java |   4 +-
 .../index/shard/StoreRecovery.java            |   3 +-
 .../index/termvectors/TermVectorsService.java |  23 +-
 .../rest/action/cat/RestAliasAction.java      |   2 +
 .../rest/action/cat/RestIndicesAction.java    |  14 +-
 .../rest/action/search/RestSearchAction.java  |   2 +-
 .../search/DefaultSearchContext.java          |   7 +-
 .../elasticsearch/search/DocValueFormat.java  |  15 +
 .../search/SearchContextMissingException.java |  17 +-
 .../search/SearchPhaseResult.java             |   9 +-
 .../elasticsearch/search/SearchService.java   |  96 +-
 .../aggregations/InternalAggregation.java     |  24 +
 .../aggregations/InternalAggregations.java    |   9 +
 .../InternalMultiBucketAggregation.java       |  19 +-
 .../InternalSingleBucketAggregation.java      |  10 +
 .../adjacency/InternalAdjacencyMatrix.java    |   3 +-
 .../bucket/composite/CompositeAggregator.java |   3 +-
 .../bucket/composite/LongValuesSource.java    |   3 +-
 .../bucket/terms/InternalTerms.java           |  58 +-
 .../aggregations/metrics/MinAggregator.java   |   9 +-
 .../pipeline/SiblingPipelineAggregator.java   |  42 +-
 .../search/dfs/DfsSearchResult.java           |   9 +-
 .../search/fetch/FetchSearchResult.java       |   9 +-
 .../search/fetch/QueryFetchSearchResult.java  |   5 +-
 .../search/fetch/ShardFetchRequest.java       |  17 +-
 .../search/fetch/ShardFetchSearchRequest.java |   3 +-
 .../subphase/highlight/PlainHighlighter.java  |  13 +-
 .../highlight/UnifiedHighlighter.java         |  14 +-
 .../internal/FilteredSearchContext.java       |   2 +-
 .../internal/InternalScrollSearchRequest.java |  16 +-
 .../search/internal/SearchContext.java        |   2 +-
 .../search/internal/SearchContextId.java      |  82 ++
 .../search/query/QuerySearchRequest.java      |  24 +-
 .../search/query/QuerySearchResult.java       |  13 +-
 .../elasticsearch/search/sort/MinAndMax.java  |   4 +-
 .../elasticsearch/search/sort/SortValue.java  |  15 +
 .../java/org/elasticsearch/tasks/Task.java    |   7 +
 .../elasticsearch/threadpool/ThreadPool.java  |   3 -
 .../transport/ProxyConnectionStrategy.java    |   6 +-
 .../ElasticsearchExceptionTests.java          |   5 +-
 .../ExceptionSerializationTests.java          |  14 +-
 .../node/stats/NodesStatsRequestTests.java    | 149 +++
 .../restore/RestoreSnapshotRequestTests.java  |  10 -
 .../indices/forcemerge/ForceMergeIT.java      |  89 ++
 .../SimulatePipelineRequestParsingTests.java  |  31 +
 .../AbstractSearchAsyncActionTests.java       |  30 +-
 .../CanMatchPreFilterSearchPhaseTests.java    |  86 +-
 .../search/ClearScrollControllerTests.java    |  28 +-
 .../action/search/CountedCollectorTests.java  |   6 +-
 .../action/search/DfsQueryPhaseTests.java     |  60 +-
 .../action/search/FetchSearchPhaseTests.java  |  71 +-
 .../action/search/MockSearchPhaseContext.java |   5 +-
 .../action/search/SearchAsyncActionTests.java |  41 +-
 .../search/SearchPhaseControllerTests.java    |  56 +-
 .../SearchProgressActionListenerIT.java       |  21 +-
 .../search/SearchScrollAsyncActionTests.java  |  72 +-
 .../search/SearchScrollRequestTests.java      |   7 +-
 .../action/search/SearchShardTests.java       |  83 ++
 .../search/TransportSearchHelperTests.java    |  36 +-
 .../cluster/NodeConnectionsServiceTests.java  | 119 ++-
 .../cluster/coordination/JoinHelperTests.java |  10 +-
 .../cluster/coordination/MessagesTests.java   |  11 +-
 .../cluster/coordination/NodeJoinTests.java   |  52 +-
 .../cluster/coordination/ZenDiscoveryIT.java  |   2 +-
 .../ComponentTemplateMetadataTests.java       |  70 ++
 .../metadata/ComponentTemplateTests.java      | 163 ++++
 .../cluster/metadata/MetaDataTests.java       |  36 +-
 .../metadata/ToAndFromJsonMetaDataTests.java  |  16 +
 .../common/util/BitArrayTests.java            |  36 +
 .../ReplicaShardAllocatorSyncIdIT.java        |   3 +-
 .../index/engine/InternalEngineTests.java     |  35 +-
 .../index/engine/NoOpEngineTests.java         |   9 +-
 .../index/mapper/DateFieldMapperTests.java    |   3 +-
 .../index/mapper/DocumentParserTests.java     |  14 +-
 .../mapper/FieldNamesFieldMapperTests.java    |   3 +-
 .../index/mapper/IpFieldMapperTests.java      |   3 +-
 .../index/mapper/KeywordFieldMapperTests.java |   4 +
 .../index/mapper/NumberFieldMapperTests.java  |   4 +-
 .../index/mapper/TypeFieldTypeTests.java      |  37 +-
 .../query/IntervalQueryBuilderTests.java      |  15 +-
 .../index/query/RangeQueryBuilderTests.java   |   7 -
 .../query/WildcardQueryBuilderTests.java      |  11 +-
 .../index/shard/IndexShardTests.java          |  18 +
 .../search/DefaultSearchContextTests.java     |  18 +-
 .../search/SearchServiceTests.java            |  84 +-
 .../search/aggregations/bucket/MissingIT.java | 184 ----
 .../composite/CompositeAggregatorTests.java   |  76 +-
 .../geogrid/GeoGridAggregatorTestCase.java    |  13 +
 .../missing/MissingAggregatorTests.java       | 687 +++++++++++---
 .../search/query/QuerySearchResultTests.java  |   6 +-
 .../search/query/SearchQueryIT.java           | 138 ++-
 .../search/scroll/SearchScrollIT.java         |  40 +
 .../snapshots/SnapshotRequestsTests.java      |   2 -
 .../RemoteClusterConnectionTests.java         |   4 +-
 .../org/elasticsearch/test/ESTestCase.java    |   6 +
 .../elasticsearch/test/TestSearchContext.java |   5 +-
 .../elasticsearch/test/XContentTestUtils.java |   7 +
 .../test/rest/ESRestTestCase.java             |   1 +
 .../authentication/custom-realm.asciidoc      |   5 +-
 .../authentication/saml-guide.asciidoc        |   2 +-
 .../topmetrics/InternalTopMetrics.java        |  97 +-
 .../topmetrics/TopMetricsAggregator.java      | 309 ++++++-
 .../TopMetricsAggregatorFactory.java          |   8 +-
 .../InternalTopMetricsReduceTests.java        |   4 +-
 .../topmetrics/InternalTopMetricsTests.java   | 138 ++-
 .../TopMetricsAggregatorMetricsTests.java     | 213 +++++
 .../topmetrics/TopMetricsAggregatorTests.java |  70 +-
 x-pack/plugin/async-search/build.gradle       |  42 +
 x-pack/plugin/async-search/qa/build.gradle    |   8 +
 .../async-search/qa/security/build.gradle     |  19 +
 .../plugin/async-search/qa/security/roles.yml |  33 +
 .../xpack/search/AsyncSearchSecurityIT.java   | 162 ++++
 .../xpack/search/AsyncSearch.java             |  92 ++
 .../xpack/search/AsyncSearchId.java           | 106 +++
 .../xpack/search/AsyncSearchIndexService.java | 342 +++++++
 .../search/AsyncSearchMaintenanceService.java | 117 +++
 .../xpack/search/AsyncSearchTask.java         | 370 ++++++++
 .../xpack/search/MutableSearchResponse.java   | 173 ++++
 .../search/RestDeleteAsyncSearchAction.java   |  40 +
 .../search/RestGetAsyncSearchAction.java      |  50 +
 .../search/RestSubmitAsyncSearchAction.java   |  71 ++
 .../TransportDeleteAsyncSearchAction.java     |  72 ++
 .../search/TransportGetAsyncSearchAction.java | 144 +++
 .../TransportSubmitAsyncSearchAction.java     | 201 ++++
 .../plugin-metadata/plugin-security.policy    |   0
 .../xpack/search/AsyncSearchActionTests.java  | 256 ++++++
 .../xpack/search/AsyncSearchIdTests.java      |  42 +
 .../search/AsyncSearchIndexServiceTests.java  | 103 +++
 .../search/AsyncSearchIntegTestCase.java      | 394 ++++++++
 .../search/AsyncSearchResponseTests.java      | 131 +++
 .../xpack/search/AsyncSearchTaskTests.java    | 182 ++++
 .../search/DeleteAsyncSearchRequestTests.java |  24 +
 .../search/GetAsyncSearchRequestTests.java    |  41 +
 .../search/SubmitAsyncSearchRequestTests.java | 112 +++
 x-pack/plugin/build.gradle                    |   2 +
 .../xpack/ccr/action/ShardChangesAction.java  |   2 +-
 .../ccr/action/ShardFollowTasksExecutor.java  |  46 +-
 .../elasticsearch/xpack/CcrIntegTestCase.java |   1 -
 .../xpack/CcrSingleNodeTestCase.java          |   1 -
 .../license/XPackLicenseState.java            |  10 +
 .../xpack/core/XPackClientPlugin.java         |  12 +-
 .../xpack/core/XPackSettings.java             |   6 +-
 .../core/beats/BeatsFeatureSetUsage.java      |  24 -
 .../xpack/core/ccr/AutoFollowStats.java       |   5 +-
 .../xpack/core/ilm/FreezeStep.java            |   8 +-
 .../ml/action/EstimateModelMemoryAction.java  | 176 ++++
 .../dataframe/analyses/BoostedTreeParams.java |  40 +-
 .../ml/dataframe/analyses/Classification.java |  67 +-
 .../dataframe/analyses/DataFrameAnalysis.java |  28 +-
 .../dataframe/analyses/OutlierDetection.java  |   2 +-
 .../ml/dataframe/analyses/Regression.java     |   2 +-
 .../ml/dataframe/evaluation/Evaluation.java   |   5 +-
 .../evaluation/EvaluationMetric.java          |   5 +-
 .../evaluation/EvaluationParameters.java      |  25 +
 .../evaluation/classification/Accuracy.java   |  19 +-
 .../MulticlassConfusionMatrix.java            |  61 +-
 .../classification/PainlessScripts.java       |  35 +
 .../evaluation/classification/Precision.java  |  14 +-
 .../evaluation/classification/Recall.java     |  16 +-
 .../regression/MeanSquaredError.java          |   5 +-
 .../evaluation/regression/RSquared.java       |   5 +-
 .../AbstractConfusionMatrixMetric.java        |   5 +-
 .../evaluation/softclassification/AucRoc.java |   5 +-
 .../core/ml/inference/TrainedModelConfig.java |  44 +-
 .../ml/inference/TrainedModelDefinition.java  |   3 +-
 .../trainedmodel/ensemble/Ensemble.java       |  11 +-
 .../persistence/AnomalyDetectorsIndex.java    |  39 +-
 .../ml/job/results/ReservedFieldNames.java    |   3 +-
 .../xpack/core/ml/utils/MlIndexAndAlias.java  | 167 ++--
 .../search/action/AsyncSearchResponse.java    | 208 +++++
 .../action/DeleteAsyncSearchAction.java       |  72 ++
 .../search/action/GetAsyncSearchAction.java   | 120 +++
 .../action/SubmitAsyncSearchAction.java       |  17 +
 .../action/SubmitAsyncSearchRequest.java      | 183 ++++
 .../action/CreateApiKeyRequestBuilder.java    |   6 +-
 .../security/action/GrantApiKeyAction.java    |  24 +
 .../security/action/GrantApiKeyRequest.java   | 166 ++++
 .../index/RestrictedIndicesNames.java         |   2 +-
 .../core/slm/SnapshotLifecyclePolicy.java     |   4 +-
 .../transforms/pivot/SingleGroupSource.java   |   4 +-
 .../state_index_template.json                 |   3 +-
 .../xpack/core/ml/config_index_mappings.json  |   7 +-
 .../core/ml/inference_index_template.json     |   3 +
 .../xpack/core/ml/size_based_ilm_policy.json  |  11 +
 .../xpack/core/ml/stats_index_template.json   |   4 +-
 .../snapshots/SourceOnlySnapshotIT.java       |  12 +-
 .../xpack/core/ilm/FreezeStepTests.java       |  31 +-
 .../DataFrameAnalyticsConfigTests.java        |   2 +
 .../analyses/BoostedTreeParamsTests.java      |  10 +-
 .../analyses/ClassificationTests.java         | 101 +-
 .../evaluation/EvaluationParametersTests.java |  18 +
 .../classification/AccuracyTests.java         |  13 +-
 .../classification/ClassificationTests.java   |   9 +-
 .../MulticlassConfusionMatrixTests.java       | 127 ++-
 .../classification/PainlessScriptsTests.java  |  19 +
 .../classification/PrecisionTests.java        |   9 +-
 .../classification/RecallTests.java           |   9 +-
 .../regression/RegressionTests.java           |   5 +-
 .../BinarySoftClassificationTests.java        |   5 +-
 .../ml/inference/TrainedModelConfigTests.java |  14 +-
 .../core/ml/utils/MlIndexAndAliasTests.java   |  72 +-
 .../deprecation/NodeDeprecationChecks.java    |  30 +
 .../NodeDeprecationChecksTests.java           |  44 +
 .../test/eql/CommonEqlRestTestCase.java       |  74 +-
 .../xpack/eql/optimizer/Optimizer.java        |  27 +
 .../xpack/eql/action/EqlActionIT.java         |  71 +-
 .../xpack/eql/optimizer/OptimizerTests.java   |  59 +-
 .../planner/AbstractQueryFolderTestCase.java  |  38 +
 .../eql/planner/QueryFolderFailTests.java     |  25 +
 .../xpack/eql/planner/QueryFolderOkTests.java | 105 +++
 .../xpack/eql/planner/QueryFolderTests.java   |  56 --
 .../resources/test_queries_unsupported.toml   | 119 +--
 .../index/engine/FrozenEngine.java            |  23 +-
 .../ilm/TimeSeriesLifecycleActionsIT.java     |   5 +-
 .../xpack/slm/SnapshotLifecycleRestIT.java    |   1 +
 .../IndexLifecycleInitialisationTests.java    |   1 -
 .../SnapshotLifecycleInitialisationTests.java |   1 -
 .../slm/SnapshotLifecyclePolicyTests.java     |  13 +
 .../xpack/logstash/Logstash.java              |  25 +-
 .../logstash/LogstashInfoTransportAction.java |   8 +-
 .../LogstashUsageTransportAction.java         |  33 +-
 .../LogstashInfoTransportActionTests.java     |  38 +-
 x-pack/plugin/ml/build.gradle                 |   1 +
 .../ml/qa/ml-with-security/build.gradle       |  11 +-
 .../ml/integration/CategorizationIT.java      |   3 +-
 .../ClassificationEvaluationIT.java           | 227 ++++-
 .../ml/integration/ClassificationIT.java      |  30 +-
 .../ml/integration/DeleteExpiredDataIT.java   |  16 +-
 .../ml/integration/InferenceIngestIT.java     |  47 +-
 .../ml/integration/MlNativeIntegTestCase.java |  13 +-
 .../xpack/ml/integration/RegressionIT.java    |  21 +-
 .../xpack/ml/MachineLearning.java             |   5 +
 .../MachineLearningUsageTransportAction.java  |   4 +
 .../xpack/ml/MlIndexTemplateRegistry.java     |  61 +-
 .../TransportEstimateModelMemoryAction.java   | 193 ++++
 .../TransportEvaluateDataFrameAction.java     |  32 +-
 ...ransportStartDataFrameAnalyticsAction.java |  45 +-
 .../scroll/TimeBasedExtractedFields.java      |   3 +-
 .../extractor/DataFrameDataExtractor.java     |   4 +
 .../extractor/ExtractedFieldsDetector.java    |  17 +-
 .../process/AnalyticsProcessConfig.java       |  30 +-
 .../process/AnalyticsProcessManager.java      |   2 +-
 .../process/AnalyticsResultProcessor.java     |  13 +-
 .../xpack/ml/extractor/ExtractedFields.java   |  14 +-
 .../xpack/ml/extractor/MultiField.java        |   2 +-
 .../inference/ingest/InferenceProcessor.java  |  30 +-
 .../inference/loadingservice/LocalModel.java  |   9 +-
 .../ml/inference/loadingservice/Model.java    |  24 +
 .../loadingservice/ModelLoadingService.java   |   6 +-
 .../job/RestEstimateModelMemoryAction.java    |  39 +
 .../MachineLearningLicensingTests.java        |   2 +-
 .../ml/MlIndexTemplateRegistryTests.java      | 181 ++++
 .../xpack/ml/MlSingleNodeTestCase.java        |   9 +-
 ...ansportEstimateModelMemoryActionTests.java | 131 +++
 ...sportGetTrainedModelsStatsActionTests.java |   2 +-
 .../DataFrameDataExtractorTests.java          |   6 +-
 .../ExtractedFieldsDetectorTests.java         |   4 +-
 .../process/AnalyticsProcessConfigTests.java  | 170 ++++
 .../process/AnalyticsProcessManagerTests.java |   4 +-
 .../AnalyticsResultProcessorTests.java        |  25 +-
 .../ml/extractor/ExtractedFieldsTests.java    |   8 +-
 .../InferenceProcessorFactoryTests.java       |  18 +-
 .../loadingservice/LocalModelTests.java       |  24 +-
 .../ml/integration/AnnotationIndexIT.java     |   8 -
 .../AutodetectResultProcessorIT.java          |   7 +-
 .../integration/BasicDistributedJobsIT.java   |   2 +-
 .../AutodetectProcessManagerTests.java        |  30 +-
 .../xpack/ml/support/BaseMlIntegTestCase.java |  11 +-
 .../extractor/AbstractFieldHitExtractor.java  |   2 +
 .../ql/expression/predicate/regex/Like.java   |  13 -
 .../predicate/regex/LikePattern.java          |   8 +-
 .../ql/expression/predicate/regex/RLike.java  |  17 +-
 .../predicate/regex/RLikePattern.java         |  20 +
 .../predicate/regex/RegexMatch.java           |  30 +-
 .../predicate/regex/StringPattern.java        |  13 +
 .../xpack/ql/index/IndexResolver.java         |  21 +-
 .../ql/planner/ExpressionTranslators.java     |   4 +-
 .../xpack/ql/type/ConstantKeywordEsField.java |  22 +
 .../xpack/ql/type/DataTypeConverter.java      |   8 +-
 .../xpack/ql/type/DataTypes.java              |  38 +-
 .../xpack/ql/type/KeywordEsField.java         |   7 +-
 .../xpack/ql/type/TextEsField.java            |   3 +-
 .../elasticsearch/xpack/ql/type/Types.java    |   3 +
 .../xpack/ql/expression/LiteralTests.java     |   3 +-
 .../ql/optimizer/OptimizerRulesTests.java     |   5 +-
 .../ql/type/DataTypeConversionTests.java      |   7 +
 .../xpack/ql/type/TypesTests.java             |  14 +-
 .../resources/mapping-constant-keyword.json   |   8 +
 .../test/resources/mapping-multi-field.json   |   4 +
 .../security/SecurityInBasicRestTestCase.java |  49 +
 .../security/SecurityWithBasicLicenseIT.java  |  23 +-
 .../security/qa/security-trial/build.gradle   |  28 +
 .../SecurityOnTrialLicenseRestTestCase.java   | 111 +++
 .../xpack/security/apikey/ApiKeyRestIT.java   | 118 +++
 .../src/test/resources/roles.yml              |   8 +
 .../xpack/security/Security.java              |   5 +
 .../action/TransportCreateApiKeyAction.java   |  32 +-
 .../action/TransportGrantApiKeyAction.java    |  88 ++
 .../xpack/security/authc/ApiKeyService.java   |   2 +-
 .../xpack/security/authc/TokenService.java    |  29 +
 .../authc/support/ApiKeyGenerator.java        |  58 ++
 .../xpack/security/authz/RBACEngine.java      |  49 +-
 .../SecuritySearchOperationListener.java      |   3 +-
 .../action/apikey/RestGrantApiKeyAction.java  |  93 ++
 .../TransportGrantApiKeyActionTests.java      | 230 +++++
 .../security/authc/TokenServiceMock.java      |  70 ++
 .../authc/support/ApiKeyGeneratorTests.java   |  86 ++
 .../SecuritySearchOperationListenerTests.java |  30 +-
 .../xpack/security/test/SecurityMocks.java    |  28 +
 .../xpack/sql/jdbc/EsDataSource.java          |   7 +-
 .../xpack/sql/jdbc/EsDriver.java              |   8 +-
 .../elasticsearch/xpack/sql/jdbc/EsType.java  |   1 +
 .../xpack/sql/jdbc/InfoResponse.java          |  19 +-
 .../xpack/sql/jdbc/JdbcConfiguration.java     |   4 +-
 .../xpack/sql/jdbc/JdbcConnection.java        |   4 +-
 .../xpack/sql/jdbc/JdbcDatabaseMetaData.java  |  14 +-
 .../xpack/sql/jdbc/JdbcHttpClient.java        |  18 +-
 .../xpack/sql/jdbc/TypeConverter.java         |   8 +
 .../xpack/sql/jdbc/TypeUtils.java             |   3 +-
 .../xpack/sql/jdbc/VersionParityTests.java    |   7 +-
 .../xpack/sql/jdbc/VersionTests.java          |   9 +-
 .../sql/qa/multi_node/RestSqlMultinodeIT.java |   3 +-
 .../sql/qa/security/RestSqlSecurityIT.java    |  22 +-
 .../xpack/sql/qa/security/UserFunctionIT.java |   4 +-
 .../xpack/sql/qa/single_node/RestSqlIT.java   |   8 +-
 .../xpack/sql/qa/FieldExtractorTestCase.java  |  34 +
 .../xpack/sql/qa/SqlProtocolTestCase.java     |  18 +-
 .../xpack/sql/qa/jdbc/DataLoader.java         |   9 +-
 .../sql/qa/rest/BaseRestSqlTestCase.java      |  11 +-
 .../xpack/sql/qa/rest/RestSqlTestCase.java    |  38 +-
 .../sql/qa/rest/RestSqlUsageTestCase.java     |  20 +-
 .../sql/qa/src/main/resources/alias.csv-spec  |   6 +-
 .../qa/src/main/resources/command.csv-spec    |   6 +-
 .../main/resources/constant-keyword.csv-spec  | 163 ++++
 .../single-node-only/command-sys.csv-spec     | 148 +--
 ...e-core-8.5.0-snapshot-7f057455901.jar.sha1 |   1 +
 ...e-core-8.5.0-snapshot-c4475920b08.jar.sha1 |   1 -
 .../sql/action/AbstractSqlQueryRequest.java   |  32 +-
 .../xpack/sql/action/AbstractSqlRequest.java  |  15 +-
 .../sql/action/SqlClearCursorRequest.java     |   2 +
 .../xpack/sql/action/SqlQueryRequest.java     |   2 +-
 .../sql/action/SqlQueryRequestBuilder.java    |   5 +
 .../xpack/sql/action/SqlTranslateRequest.java |  12 +-
 .../sql/action/SqlQueryRequestTests.java      |   3 +
 .../sql/action/SqlRequestParsersTests.java    |  20 +-
 .../org/elasticsearch/xpack/sql/cli/Cli.java  |   4 +-
 .../xpack/sql/cli/command/CliSession.java     |  15 +-
 .../sql/cli/command/PrintLogoCommand.java     |   6 +-
 .../xpack/sql/cli/CliSessionTests.java        |  38 +-
 .../xpack/sql/cli/VersionTests.java           |  10 +-
 .../sql/cli/command/BuiltinCommandTests.java  |   4 +-
 .../{Version.java => ClientVersion.java}      |  76 +-
 .../sql/client/ConnectionConfiguration.java   |   2 -
 .../xpack/sql/client/HttpClient.java          |   2 +-
 .../sql/client/HttpClientRequestTests.java    |   2 +-
 .../xpack/sql/client/VersionTests.java        |  42 +-
 .../xpack/sql/proto/AbstractSqlRequest.java   |   4 +
 .../elasticsearch/xpack/sql/proto/Mode.java   |   4 +
 .../xpack/sql/proto/RequestInfo.java          |  23 +-
 .../sql/proto/SqlClearCursorRequest.java      |   3 +
 .../xpack/sql/proto/SqlQueryRequest.java      |   3 +
 .../xpack/sql/proto/SqlVersion.java           | 144 +++
 .../xpack/sql/proto/SqlVersionTests.java      |  63 ++
 .../xpack/sql/parser/ExpressionBuilder.java   |   6 +-
 .../xpack/sql/plugin/RestSqlQueryAction.java  |   2 +-
 .../xpack/sql/type/SqlDataTypeConverter.java  |   3 +-
 .../xpack/sql/type/SqlDataTypes.java          |  11 +
 .../xpack/sql/action/SqlActionIT.java         |  17 +-
 .../analysis/index/IndexResolverTests.java    |  51 +-
 .../xpack/sql/optimizer/OptimizerTests.java   |   3 +-
 .../xpack/sql/parser/ExpressionTests.java     |  10 +
 .../logical/command/sys/SysTypesTests.java    |   6 +-
 .../sql/planner/QueryTranslatorTests.java     |  17 +
 .../sql/type/SqlDataTypeConverterTests.java   |   7 +
 .../test/rest/CatIndicesWithSecurityIT.java   | 127 +++
 .../xpack/test/rest/XPackRestIT.java          |   9 +-
 .../api/async_search.delete.json              |  24 +
 .../rest-api-spec/api/async_search.get.json   |  40 +
 .../api/async_search.submit.json              | 225 +++++
 .../api/ml.estimate_model_memory.json         |  23 +
 .../test/async_search/10_basic.yml            | 143 +++
 .../test/ml/data_frame_analytics_crud.yml     |  86 +-
 .../test/ml/estimate_model_memory.yml         | 172 ++++
 .../rest-api-spec/test/ml/index_layout.yml    |  10 +-
 .../rest-api-spec/test/ml/inference_crud.yml  |   2 +-
 .../test/ml/inference_processor.yml           | 114 +++
 .../test/ml/inference_stats_crud.yml          |   4 +-
 .../rest-api-spec/test/ml/jobs_crud.yml       |   8 +-
 .../test/ml/start_data_frame_analytics.yml    |  21 +-
 .../test/wildcard/10_wildcard_basic.yml       | 218 +++++
 x-pack/plugin/src/test/resources/roles.yml    |   7 +
 .../watcher/WatcherYamlSuiteTestCase.java     |  10 +-
 x-pack/plugin/wildcard/build.gradle           |  18 +
 .../xpack/wildcard/Wildcard.java              |  31 +
 .../mapper/AutomatonQueryOnBinaryDv.java      | 104 +++
 .../wildcard/mapper/WildcardFieldMapper.java  | 575 ++++++++++++
 .../mapper/WildcardFieldMapperTests.java      | 331 +++++++
 .../mapper/WildcardFieldTypeTests.java        |  19 +
 ...nfigIndexMappingsFullClusterRestartIT.java |  11 +-
 .../90_ml_data_frame_analytics_crud.yml       |  30 +
 .../90_ml_data_frame_analytics_crud.yml       |  36 +
 .../90_ml_data_frame_analytics_crud.yml       |  31 +
 684 files changed, 20435 insertions(+), 6014 deletions(-)
 create mode 100644 docs/reference/images/analysis/token-graph-basic.svg
 create mode 100644 docs/reference/images/analysis/token-graph-wd.svg
 create mode 100644 docs/reference/images/analysis/token-graph-wdg.svg
 create mode 100644 docs/reference/mapping/types/wildcard.asciidoc
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
 create mode 100644 server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1
 create mode 100644 server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1
 delete mode 100644 server/src/main/java/org/apache/lucene/queries/XIntervals.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
 create mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java
 create mode 100644 x-pack/plugin/async-search/build.gradle
 create mode 100644 x-pack/plugin/async-search/qa/build.gradle
 create mode 100644 x-pack/plugin/async-search/qa/security/build.gradle
 create mode 100644 x-pack/plugin/async-search/qa/security/roles.yml
 create mode 100644 x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
 create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
 create mode 100644 x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java
 create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
 create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
 create mode 100644 x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
 create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java
 create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
 create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
 delete mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
 create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
 create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
 create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
 create mode 100644 x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
 create mode 100644 x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java
 create mode 100644 x-pack/plugin/security/qa/security-trial/build.gradle
 create mode 100644 x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
 create mode 100644 x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
 create mode 100644 x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java
 create mode 100644 x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
 rename x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/{Version.java => ClientVersion.java} (57%)
 create mode 100644 x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java
 create mode 100644 x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java
 create mode 100644 x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
 create mode 100644 x-pack/plugin/src/test/resources/roles.yml
 create mode 100644 x-pack/plugin/wildcard/build.gradle
 create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
 create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
 create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
 create mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
 create mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java

diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
index 4a0065b27afd4..b8c543cea0b08 100644
--- a/.ci/matrix-runtime-javas.yml
+++ b/.ci/matrix-runtime-javas.yml
@@ -7,7 +7,6 @@
 
 ES_RUNTIME_JAVA:
   - java11
-  - openjdk13
   - openjdk14
   - openjdk15
   - zulu11
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1f1a4720a2dd8..eeea7b412cbf8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -171,13 +171,13 @@ IntelliJ or Eclipse like describe above to use
 Elasticsearch typically uses singular nouns rather than plurals in URLs.
 For example:
 
-    /_ingest/pipline
-    /_ingest/pipline/{id}
+    /_ingest/pipeline
+    /_ingest/pipeline/{id}
 
 but not:
 
-    /_ingest/piplines
-    /_ingest/piplines/{id}
+    /_ingest/pipelines
+    /_ingest/pipelines/{id}
 
 You may find counterexamples, but new endpoints should use the singular
 form.
diff --git a/build.gradle b/build.gradle
index 5cee1f108cd69..78322b938e6f4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -118,7 +118,8 @@ subprojects {
       ':qa:os',
       ':qa:wildfly',
       ':x-pack:plugin:autoscaling',
-      ':x-pack:plugin:enrich'
+      ':x-pack:plugin:enrich',
+      ':x-pack:plugin:logstash'
     ]
 
     if (projectPathsToFormat.contains(project.path)) {
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
index 63aedd6566d89..0982fe0267cc5 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -174,24 +174,21 @@ private void setupRootDownload(Project rootProject, ElasticsearchDistribution di
     }
 
     private static void addIvyRepo(Project project, String name, String url, String group) {
-        project.getRepositories().ivy(ivyRepo -> {
-            ivyRepo.setName(name);
-            ivyRepo.setUrl(url);
-            ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+        IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
+            repo.setName(name);
+            repo.setUrl(url);
+            repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
             // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
-            ivyRepo.credentials(HttpHeaderCredentials.class, creds -> {
+            repo.credentials(HttpHeaderCredentials.class, creds -> {
                 creds.setName("X-Elastic-No-KPI");
                 creds.setValue("1");
             });
-            ivyRepo.getAuthentication().create("header", HttpHeaderAuthentication.class);
-            ivyRepo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
-            ivyRepo.content(content -> content.includeGroup(group));
+            repo.getAuthentication().create("header", HttpHeaderAuthentication.class);
+            repo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
         });
-        project.getRepositories().all(repo -> {
-            if (repo.getName().equals(name) == false) {
-                // all other repos should ignore the special group name
-                repo.content(content -> content.excludeGroup(group));
-            }
+        project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
+            exclusiveContentRepository.filter(config -> config.includeGroup(group));
+            exclusiveContentRepository.forRepositories(ivyRepo);
         });
     }
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
index 9c48fd8ac1b19..b63d8f65ea04f 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
@@ -79,16 +79,6 @@ public void apply(Project project) {
                 setupRootJdkDownload(project.getRootProject(), jdk);
             }
         });
-
-        // all other repos should ignore the special jdk artifacts
-        project.getRootProject().getRepositories().all(repo -> {
-            if (repo.getName().startsWith(REPO_NAME_PREFIX) == false) {
-                repo.content(content -> {
-                    content.excludeGroup("adoptopenjdk");
-                    content.excludeGroup("openjdk");
-                });
-            }
-        });
     }
 
     @SuppressWarnings("unchecked")
@@ -145,13 +135,16 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
             }
 
             // Define the repository if we haven't already
-            if (rootProject.getRepositories().findByName(repoName) == null) {
-                repositories.ivy(ivyRepo -> {
-                    ivyRepo.setName(repoName);
-                    ivyRepo.setUrl(repoUrl);
-                    ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
-                    ivyRepo.patternLayout(layout -> layout.artifact(artifactPattern));
-                    ivyRepo.content(content -> content.includeGroup(jdk.getVendor()));
+            if (repositories.findByName(repoName) == null) {
+                IvyArtifactRepository ivyRepo = repositories.ivy(repo -> {
+                    repo.setName(repoName);
+                    repo.setUrl(repoUrl);
+                    repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                    repo.patternLayout(layout -> layout.artifact(artifactPattern));
+                });
+                repositories.exclusiveContent(exclusiveContentRepository -> {
+                    exclusiveContentRepository.filter(config -> config.includeGroup(groupName(jdk)));
+                    exclusiveContentRepository.forRepositories(ivyRepo);
                 });
             }
 
@@ -261,7 +254,11 @@ private static String dependencyNotation(Jdk jdk) {
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 
-        return jdk.getVendor() + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+    }
+
+    private static String groupName(Jdk jdk) {
+        return jdk.getVendor() + "_" + jdk.getMajor();
     }
 
     private static String configName(String... parts) {
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 9d654e0e6682b..dea67de1657d1 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.5.0-snapshot-c4475920b08
+lucene            = 8.5.0-snapshot-7f057455901
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.2+8
@@ -11,7 +11,7 @@ jts               = 1.15.0
 # you should also inspect that version to see if it can be advanced along with
 # the com.maxmind.geoip2:geoip2 dependency
 jackson           = 2.8.11
-jacksondatabind   = 2.8.11.4
+jacksondatabind   = 2.8.11.6
 snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java
index 2509dcb767449..12f3a64e63cb2 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java
@@ -24,12 +24,10 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -88,9 +86,9 @@ public static class TopMetrics implements ToXContent {
         private static final ParseField METRICS_FIELD = new ParseField("metrics");
 
         private final List<Object> sort;
-        private final Map<String, Double> metrics;
+        private final Map<String, Object> metrics;
 
-        private TopMetrics(List<Object> sort, Map<String, Double> metrics) {
+        private TopMetrics(List<Object> sort, Map<String, Object> metrics) {
             this.sort = sort;
             this.metrics = metrics;
         }
@@ -105,7 +103,7 @@ public List<Object> getSort() {
         /**
          * The top metric values returned by the aggregation.
          */
-        public Map<String, Double> getMetrics() {
+        public Map<String, Object> getMetrics() {
             return metrics;
         }
 
@@ -114,13 +112,13 @@ public Map<String, Double> getMetrics() {
                     @SuppressWarnings("unchecked")
                     List<Object> sort = (List<Object>) args[0];
                     @SuppressWarnings("unchecked")
-                    Map<String, Double> metrics = (Map<String, Double>) args[1];
+                    Map<String, Object> metrics = (Map<String, Object>) args[1];
                     return new TopMetrics(sort, metrics);
                 });
         static {
             PARSER.declareFieldArray(constructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p),
                     SORT_FIELD, ObjectParser.ValueType.VALUE_ARRAY);
-            PARSER.declareObject(constructorArg(), (p, c) -> p.map(HashMap::new, XContentParser::doubleValue), METRICS_FIELD);
+            PARSER.declareObject(constructorArg(), (p, c) -> p.map(), METRICS_FIELD);
         }
 
         public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
index 0fc4f240eb8ef..79758ec709c63 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
@@ -23,9 +23,9 @@
 
 public class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
     static final String NAME = "proxy";
-    static final String ADDRESS = "address";
-    static final String NUM_SOCKETS_CONNECTED = "num_sockets_connected";
-    static final String MAX_SOCKET_CONNECTIONS = "max_socket_connections";
+    static final String PROXY_ADDRESS = "proxy_address";
+    static final String NUM_PROXY_SOCKETS_CONNECTED = "num_proxy_sockets_connected";
+    static final String MAX_PROXY_SOCKET_CONNECTIONS = "max_proxy_socket_connections";
     private final String address;
     private final int maxSocketConnections;
     private final int numSocketsConnected;
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
index 2bf99c61085c4..474b991cd434a 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
@@ -66,9 +66,9 @@ public final class RemoteConnectionInfo {
         PARSER.declareString(constructorArg(), new ParseField(INITIAL_CONNECT_TIMEOUT));
         PARSER.declareBoolean(constructorArg(), new ParseField(SKIP_UNAVAILABLE));
 
-        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.ADDRESS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_SOCKET_CONNECTIONS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_SOCKETS_CONNECTED));
+        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.PROXY_ADDRESS));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_PROXY_SOCKET_CONNECTIONS));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_PROXY_SOCKETS_CONNECTED));
 
         PARSER.declareStringArray(optionalConstructorArg(), new ParseField(SniffModeInfo.SEEDS));
         PARSER.declareInt(optionalConstructorArg(), new ParseField(SniffModeInfo.MAX_CONNECTIONS_PER_CLUSTER));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
index 02861adc73845..48c75e6e34e6d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
@@ -22,10 +22,12 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Locale;
 import java.util.Objects;
 
 public class Classification implements DataFrameAnalysis {
@@ -44,11 +46,12 @@ public static Builder builder(String dependentVariable) {
     static final ParseField LAMBDA = new ParseField("lambda");
     static final ParseField GAMMA = new ParseField("gamma");
     static final ParseField ETA = new ParseField("eta");
-    static final ParseField MAXIMUM_NUMBER_TREES = new ParseField("maximum_number_trees");
+    static final ParseField MAX_TREES = new ParseField("max_trees");
     static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
     static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
     static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
+    static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
     static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
     static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
@@ -67,47 +70,57 @@ public static Builder builder(String dependentVariable) {
                 (String) a[7],
                 (Double) a[8],
                 (Integer) a[9],
-                (Long) a[10]));
+                (Long) a[10],
+                (ClassAssignmentObjective) a[11]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DEPENDENT_VARIABLE);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), LAMBDA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), GAMMA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), ETA);
-        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_NUMBER_TREES);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_TREES);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), FEATURE_BAG_FRACTION);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_FEATURE_IMPORTANCE_VALUES);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PREDICTION_FIELD_NAME);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), TRAINING_PERCENT);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_CLASSES);
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), RANDOMIZE_SEED);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ClassAssignmentObjective.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
     }
 
     private final String dependentVariable;
     private final Double lambda;
     private final Double gamma;
     private final Double eta;
-    private final Integer maximumNumberTrees;
+    private final Integer maxTrees;
     private final Double featureBagFraction;
     private final Integer numTopFeatureImportanceValues;
     private final String predictionFieldName;
     private final Double trainingPercent;
+    private final ClassAssignmentObjective classAssignmentObjective;
     private final Integer numTopClasses;
     private final Long randomizeSeed;
 
     private Classification(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
-                           @Nullable Integer maximumNumberTrees, @Nullable Double featureBagFraction,
+                           @Nullable Integer maxTrees, @Nullable Double featureBagFraction,
                            @Nullable Integer numTopFeatureImportanceValues, @Nullable String predictionFieldName,
-                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed) {
+                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed,
+                           @Nullable ClassAssignmentObjective classAssignmentObjective) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
         this.eta = eta;
-        this.maximumNumberTrees = maximumNumberTrees;
+        this.maxTrees = maxTrees;
         this.featureBagFraction = featureBagFraction;
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
         this.predictionFieldName = predictionFieldName;
         this.trainingPercent = trainingPercent;
+        this.classAssignmentObjective = classAssignmentObjective;
         this.numTopClasses = numTopClasses;
         this.randomizeSeed = randomizeSeed;
     }
@@ -133,8 +146,8 @@ public Double getEta() {
         return eta;
     }
 
-    public Integer getMaximumNumberTrees() {
-        return maximumNumberTrees;
+    public Integer getMaxTrees() {
+        return maxTrees;
     }
 
     public Double getFeatureBagFraction() {
@@ -157,6 +170,10 @@ public Long getRandomizeSeed() {
         return randomizeSeed;
     }
 
+    public ClassAssignmentObjective getClassAssignmentObjective() {
+        return classAssignmentObjective;
+    }
+
     public Integer getNumTopClasses() {
         return numTopClasses;
     }
@@ -174,8 +191,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (eta != null) {
             builder.field(ETA.getPreferredName(), eta);
         }
-        if (maximumNumberTrees != null) {
-            builder.field(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
+        if (maxTrees != null) {
+            builder.field(MAX_TREES.getPreferredName(), maxTrees);
         }
         if (featureBagFraction != null) {
             builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -192,6 +209,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (randomizeSeed != null) {
             builder.field(RANDOMIZE_SEED.getPreferredName(), randomizeSeed);
         }
+        if (classAssignmentObjective != null) {
+            builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
+        }
         if (numTopClasses != null) {
             builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         }
@@ -201,8 +221,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues,
-            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses);
+        return Objects.hash(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues,
+            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses, classAssignmentObjective);
     }
 
     @Override
@@ -214,13 +234,14 @@ public boolean equals(Object o) {
             && Objects.equals(lambda, that.lambda)
             && Objects.equals(gamma, that.gamma)
             && Objects.equals(eta, that.eta)
-            && Objects.equals(maximumNumberTrees, that.maximumNumberTrees)
+            && Objects.equals(maxTrees, that.maxTrees)
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(numTopFeatureImportanceValues, that.numTopFeatureImportanceValues)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
             && Objects.equals(trainingPercent, that.trainingPercent)
             && Objects.equals(randomizeSeed, that.randomizeSeed)
-            && Objects.equals(numTopClasses, that.numTopClasses);
+            && Objects.equals(numTopClasses, that.numTopClasses)
+            && Objects.equals(classAssignmentObjective, that.classAssignmentObjective);
     }
 
     @Override
@@ -228,18 +249,32 @@ public String toString() {
         return Strings.toString(this);
     }
 
+    public enum ClassAssignmentObjective {
+        MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
+
+        public static ClassAssignmentObjective fromString(String value) {
+            return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
     public static class Builder {
         private String dependentVariable;
         private Double lambda;
         private Double gamma;
         private Double eta;
-        private Integer maximumNumberTrees;
+        private Integer maxTrees;
         private Double featureBagFraction;
         private Integer numTopFeatureImportanceValues;
         private String predictionFieldName;
         private Double trainingPercent;
         private Integer numTopClasses;
         private Long randomizeSeed;
+        private ClassAssignmentObjective classAssignmentObjective;
 
         private Builder(String dependentVariable) {
             this.dependentVariable = Objects.requireNonNull(dependentVariable);
@@ -260,8 +295,8 @@ public Builder setEta(Double eta) {
             return this;
         }
 
-        public Builder setMaximumNumberTrees(Integer maximumNumberTrees) {
-            this.maximumNumberTrees = maximumNumberTrees;
+        public Builder setMaxTrees(Integer maxTrees) {
+            this.maxTrees = maxTrees;
             return this;
         }
 
@@ -295,9 +330,15 @@ public Builder setNumTopClasses(Integer numTopClasses) {
             return this;
         }
 
+        public Builder setClassAssignmentObjective(ClassAssignmentObjective classAssignmentObjective) {
+            this.classAssignmentObjective = classAssignmentObjective;
+            return this;
+        }
+
         public Classification build() {
-            return new Classification(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction,
-                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed);
+            return new Classification(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction,
+                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed,
+                classAssignmentObjective);
         }
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java
index d7e374a2563a1..7eb1af3405486 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java
@@ -44,7 +44,7 @@ public static Builder builder(String dependentVariable) {
     static final ParseField LAMBDA = new ParseField("lambda");
     static final ParseField GAMMA = new ParseField("gamma");
     static final ParseField ETA = new ParseField("eta");
-    static final ParseField MAXIMUM_NUMBER_TREES = new ParseField("maximum_number_trees");
+    static final ParseField MAX_TREES = new ParseField("max_trees");
     static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
     static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
@@ -72,7 +72,7 @@ public static Builder builder(String dependentVariable) {
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), LAMBDA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), GAMMA);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), ETA);
-        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_NUMBER_TREES);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAX_TREES);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), FEATURE_BAG_FRACTION);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_FEATURE_IMPORTANCE_VALUES);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PREDICTION_FIELD_NAME);
@@ -84,7 +84,7 @@ public static Builder builder(String dependentVariable) {
     private final Double lambda;
     private final Double gamma;
     private final Double eta;
-    private final Integer maximumNumberTrees;
+    private final Integer maxTrees;
     private final Double featureBagFraction;
     private final Integer numTopFeatureImportanceValues;
     private final String predictionFieldName;
@@ -92,14 +92,14 @@ public static Builder builder(String dependentVariable) {
     private final Long randomizeSeed;
 
     private Regression(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
-                       @Nullable Integer maximumNumberTrees, @Nullable Double featureBagFraction,
+                       @Nullable Integer maxTrees, @Nullable Double featureBagFraction,
                        @Nullable Integer numTopFeatureImportanceValues, @Nullable String predictionFieldName,
                        @Nullable Double trainingPercent, @Nullable Long randomizeSeed) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
         this.eta = eta;
-        this.maximumNumberTrees = maximumNumberTrees;
+        this.maxTrees = maxTrees;
         this.featureBagFraction = featureBagFraction;
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
         this.predictionFieldName = predictionFieldName;
@@ -128,8 +128,8 @@ public Double getEta() {
         return eta;
     }
 
-    public Integer getMaximumNumberTrees() {
-        return maximumNumberTrees;
+    public Integer getMaxTrees() {
+        return maxTrees;
     }
 
     public Double getFeatureBagFraction() {
@@ -165,8 +165,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (eta != null) {
             builder.field(ETA.getPreferredName(), eta);
         }
-        if (maximumNumberTrees != null) {
-            builder.field(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
+        if (maxTrees != null) {
+            builder.field(MAX_TREES.getPreferredName(), maxTrees);
         }
         if (featureBagFraction != null) {
             builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -189,7 +189,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues,
+        return Objects.hash(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues,
             predictionFieldName, trainingPercent, randomizeSeed);
     }
 
@@ -202,7 +202,7 @@ public boolean equals(Object o) {
             && Objects.equals(lambda, that.lambda)
             && Objects.equals(gamma, that.gamma)
             && Objects.equals(eta, that.eta)
-            && Objects.equals(maximumNumberTrees, that.maximumNumberTrees)
+            && Objects.equals(maxTrees, that.maxTrees)
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(numTopFeatureImportanceValues, that.numTopFeatureImportanceValues)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
@@ -220,7 +220,7 @@ public static class Builder {
         private Double lambda;
         private Double gamma;
         private Double eta;
-        private Integer maximumNumberTrees;
+        private Integer maxTrees;
         private Double featureBagFraction;
         private Integer numTopFeatureImportanceValues;
         private String predictionFieldName;
@@ -246,8 +246,8 @@ public Builder setEta(Double eta) {
             return this;
         }
 
-        public Builder setMaximumNumberTrees(Integer maximumNumberTrees) {
-            this.maximumNumberTrees = maximumNumberTrees;
+        public Builder setMaxTrees(Integer maxTrees) {
+            this.maxTrees = maxTrees;
             return this;
         }
 
@@ -277,7 +277,7 @@ public Builder setRandomizeSeed(Long randomizeSeed) {
         }
 
         public Regression build() {
-            return new Regression(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction,
+            return new Regression(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction,
                 numTopFeatureImportanceValues, predictionFieldName, trainingPercent, randomizeSeed);
         }
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java
index 9d2b323cf4880..d8749c0a6cdc9 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java
@@ -53,6 +53,7 @@ public class TrainedModelConfig implements ToXContentObject {
     public static final ParseField ESTIMATED_HEAP_MEMORY_USAGE_BYTES = new ParseField("estimated_heap_memory_usage_bytes");
     public static final ParseField ESTIMATED_OPERATIONS = new ParseField("estimated_operations");
     public static final ParseField LICENSE_LEVEL = new ParseField("license_level");
+    public static final ParseField DEFAULT_FIELD_MAP = new ParseField("default_field_map");
 
     public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(NAME,
             true,
@@ -76,6 +77,7 @@ public class TrainedModelConfig implements ToXContentObject {
         PARSER.declareLong(TrainedModelConfig.Builder::setEstimatedHeapMemory, ESTIMATED_HEAP_MEMORY_USAGE_BYTES);
         PARSER.declareLong(TrainedModelConfig.Builder::setEstimatedOperations, ESTIMATED_OPERATIONS);
         PARSER.declareString(TrainedModelConfig.Builder::setLicenseLevel, LICENSE_LEVEL);
+        PARSER.declareObject(TrainedModelConfig.Builder::setDefaultFieldMap, (p, c) -> p.mapStrings(), DEFAULT_FIELD_MAP);
     }
 
     public static TrainedModelConfig fromXContent(XContentParser parser) throws IOException {
@@ -95,6 +97,7 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
     private final Long estimatedHeapMemory;
     private final Long estimatedOperations;
     private final String licenseLevel;
+    private final Map<String, String> defaultFieldMap;
 
     TrainedModelConfig(String modelId,
                        String createdBy,
@@ -108,7 +111,8 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
                        TrainedModelInput input,
                        Long estimatedHeapMemory,
                        Long estimatedOperations,
-                       String licenseLevel) {
+                       String licenseLevel,
+                       Map<String, String> defaultFieldMap) {
         this.modelId = modelId;
         this.createdBy = createdBy;
         this.version = version;
@@ -122,6 +126,7 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
         this.estimatedHeapMemory = estimatedHeapMemory;
         this.estimatedOperations = estimatedOperations;
         this.licenseLevel = licenseLevel;
+        this.defaultFieldMap = defaultFieldMap == null ? null : Collections.unmodifiableMap(defaultFieldMap);
     }
 
     public String getModelId() {
@@ -180,6 +185,10 @@ public String getLicenseLevel() {
         return licenseLevel;
     }
 
+    public Map<String, String> getDefaultFieldMap() {
+        return defaultFieldMap;
+    }
+
     public static Builder builder() {
         return new Builder();
     }
@@ -226,6 +235,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (licenseLevel != null) {
             builder.field(LICENSE_LEVEL.getPreferredName(), licenseLevel);
         }
+        if (defaultFieldMap != null) {
+            builder.field(DEFAULT_FIELD_MAP.getPreferredName(), defaultFieldMap);
+        }
         builder.endObject();
         return builder;
     }
@@ -252,6 +264,7 @@ public boolean equals(Object o) {
             Objects.equals(estimatedHeapMemory, that.estimatedHeapMemory) &&
             Objects.equals(estimatedOperations, that.estimatedOperations) &&
             Objects.equals(licenseLevel, that.licenseLevel) &&
+            Objects.equals(defaultFieldMap, that.defaultFieldMap) &&
             Objects.equals(metadata, that.metadata);
     }
 
@@ -269,7 +282,8 @@ public int hashCode() {
             estimatedOperations,
             metadata,
             licenseLevel,
-            input);
+            input,
+            defaultFieldMap);
     }
 
 
@@ -288,6 +302,7 @@ public static class Builder {
         private Long estimatedHeapMemory;
         private Long estimatedOperations;
         private String licenseLevel;
+        private Map<String, String> defaultFieldMap;
 
         public Builder setModelId(String modelId) {
             this.modelId = modelId;
@@ -367,6 +382,11 @@ private Builder setLicenseLevel(String licenseLevel) {
             return this;
         }
 
+        public Builder setDefaultFieldMap(Map<String, String> defaultFieldMap) {
+            this.defaultFieldMap = defaultFieldMap;
+            return this;
+        }
+
         public TrainedModelConfig build() {
             return new TrainedModelConfig(
                 modelId,
@@ -381,7 +401,8 @@ public TrainedModelConfig build() {
                 input,
                 estimatedHeapMemory,
                 estimatedOperations,
-                licenseLevel);
+                licenseLevel,
+                defaultFieldMap);
         }
     }
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
index e01d08d019fdc..1f6eed5b3fac1 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
@@ -43,9 +43,8 @@ public class TrainedModelDefinition implements ToXContentObject {
             true,
             TrainedModelDefinition.Builder::new);
     static {
-        PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
+        PARSER.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
             (p, c, n) -> p.namedObject(TrainedModel.class, n, null),
-            (modelDocBuilder) -> { /* Noop does not matter client side*/ },
             TRAINED_MODEL);
         PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
             (p, c, n) -> p.namedObject(PreProcessor.class, n, null),
@@ -124,11 +123,6 @@ public Builder setTrainedModel(TrainedModel trainedModel) {
             return this;
         }
 
-        private Builder setTrainedModel(List<TrainedModel> trainedModel) {
-            assert trainedModel.size() == 1;
-            return setTrainedModel(trainedModel.get(0));
-        }
-
         public TrainedModelDefinition build() {
             return new TrainedModelDefinition(this.trainedModel, this.preProcessors);
         }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
index 45d8afb32e6b9..4f16422cc7bd7 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -56,9 +56,8 @@ public class Ensemble implements TrainedModel {
                     p.namedObject(TrainedModel.class, n, null),
             (ensembleBuilder) -> { /* Noop does not matter client side */ },
             TRAINED_MODELS);
-        PARSER.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
+        PARSER.declareNamedObject(Ensemble.Builder::setOutputAggregator,
             (p, c, n) -> p.namedObject(OutputAggregator.class, n, null),
-            (ensembleBuilder) -> { /* Noop does not matter client side */ },
             AGGREGATE_OUTPUT);
         PARSER.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
         PARSER.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -194,9 +193,6 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
             return this;
         }
 
-        private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
-            this.setOutputAggregator(outputAggregators.get(0));
-        }
 
         private void setTargetType(String targetType) {
             this.targetType = TargetType.fromString(targetType);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java
index 10e75d88307b6..92ed0ea83073c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java
@@ -21,6 +21,12 @@
 
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.eql.EqlSearchRequest;
 import org.elasticsearch.client.eql.EqlSearchResponse;
 import org.elasticsearch.client.eql.EqlStatsRequest;
@@ -28,51 +34,132 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.Map;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
 public class EqlIT extends ESRestHighLevelClientTestCase {
 
+    private static final String INDEX_NAME = "index";
+    private static final int RECORD_COUNT = 40;
+    private static final int DIVIDER = 4;
+
     @Before
-    public void setupRemoteClusterConfig() throws Exception {
+    public void setup() throws Exception {
         setupRemoteClusterConfig("local_cluster");
+        setupData();
     }
 
-    public void testBasicSearch() throws Exception {
-        Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
-        doc1.setJsonEntity("{\"event_subtype_full\": \"already_running\", " +
-                "\"event\": {" +
-                    "\"category\": \"process\"" +
-                "}," +
-                "\"event_type_full\": \"process_event\", " +
-                "\"opcode\": 3," +
-                "\"pid\": 0," +
-                "\"process_name\": \"System Idle Process\"," +
-                "\"serial_event_id\": 1," +
-                "\"subtype\": \"create\"," +
-                "\"@timestamp\": 116444736000000000," +
-                "\"unique_pid\": 1}");
-        client().performRequest(doc1);
-        client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh"));
+    private void setupData() throws IOException {
+        final BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        for (int i = 0; i < RECORD_COUNT; i++) {
+            final IndexRequest indexRequest = new IndexRequest(INDEX_NAME);
+            indexRequest.source(jsonBuilder()
+                    .startObject()
+                    .field("event_subtype_full", "already_running")
+                    .startObject("event")
+                    .field("category", "process")
+                    .endObject()
+                    .field("event_type", "foo")
+                    .field("event_type_full", "process_event")
+                    .field("opcode", ((i % DIVIDER) == 0) ? 1 : 0)
+                    .field("pid", ((i % DIVIDER) == 0) ? 100 : 0)
+                    .field("process_name", "System Idle Process")
+                    .field("serial_event_id", i + 1)
+                    .field("subtype", "create")
+                    .field("@timestamp", String.format(Locale.ROOT, "2018-01-01T00:00:%02dZ", i))
+                    .field("unique_pid", ((i % DIVIDER) == 0) ? 101 : 0)
+                    .endObject());
+            bulkRequest.add(indexRequest);
+        }
+        BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT);
+        assertEquals(RestStatus.OK, bulkResponse.status());
+        assertFalse(bulkResponse.hasFailures());
 
-        EqlClient eql = highLevelClient().eql();
-        EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
-        EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
+        RefreshResponse refreshResponse = highLevelClient().indices().refresh(new RefreshRequest(INDEX_NAME), RequestOptions.DEFAULT);
+        assertEquals(0, refreshResponse.getFailedShards());
+    }
+
+    private void assertResponse(EqlSearchResponse response, int count) {
         assertNotNull(response);
         assertFalse(response.isTimeout());
         assertNotNull(response.hits());
         assertNull(response.hits().sequences());
         assertNull(response.hits().counts());
         assertNotNull(response.hits().events());
-        assertThat(response.hits().events().size(), equalTo(1));
+        assertThat(response.hits().events().size(), equalTo(count));
+    }
+
+    public void testBasicSearch() throws Exception {
+        EqlClient eql = highLevelClient().eql();
+        EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
+        assertResponse(execute(request, eql::search, eql::searchAsync), RECORD_COUNT);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testSimpleConditionSearch() throws Exception {
+        EqlClient eql = highLevelClient().eql();
+
+        // test simple conditional
+        EqlSearchRequest request = new EqlSearchRequest("index", "foo where pid > 0");
+
+        // test with non-default event.category mapping
+        request.eventCategoryField("event_type");
+
+        EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
+        assertResponse(response, RECORD_COUNT / DIVIDER);
+
+        // test the content of the hits
+        for (SearchHit hit : response.hits().events()) {
+            final Map<String, Object> source = hit.getSourceAsMap();
+
+            final Map<String, Object> event = (Map<String, Object>) source.get("event");
+            assertThat(event.get("category"), equalTo("process"));
+            assertThat(source.get("event_type"), equalTo("foo"));
+            assertThat(source.get("event_type_full"), equalTo("process_event"));
+            assertThat(source.get("opcode"), equalTo(1));
+            assertThat(source.get("pid"), equalTo(100));
+            assertThat(source.get("process_name"), equalTo("System Idle Process"));
+            assertThat((int) source.get("serial_event_id"), greaterThan(0));
+            assertThat(source.get("unique_pid"), equalTo(101));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testEqualsInFilterConditionSearch() throws Exception {
+        EqlClient eql = highLevelClient().eql();
+
+        EqlSearchRequest request = new EqlSearchRequest("index",
+                "process where event_type_full = \"process_event\" and serial_event_id in (1,3,5)");
+
+        EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
+        assertResponse(response, 3);
+
+        // test the content of the hits
+        for (SearchHit hit : response.hits().events()) {
+            final Map<String, Object> source = hit.getSourceAsMap();
+
+            final Map<String, Object> event = (Map<String, Object>) source.get("event");
+            assertThat(event.get("category"), equalTo("process"));
+            assertThat(source.get("serial_event_id"), anyOf(equalTo(1), equalTo(3), equalTo(5)));
+        }
     }
 
     public void testLargeMapping() throws Exception {
-        Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
+        final String index = "large_mapping_index";
+
+        Request doc1 = new Request(HttpPut.METHOD_NAME, "/" + index + "/_doc/1");
         // use more exact fields (dates) than the default to verify that retrieval works and requesting doc values
         // would fail
         int PASS_DEFAULT_DOC_VALUES = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY) + 50;
@@ -94,7 +181,7 @@ public void testLargeMapping() throws Exception {
 
 
         EqlClient eql = highLevelClient().eql();
-        EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
+        EqlSearchRequest request = new EqlSearchRequest(index, "process where true");
         EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
         assertNotNull(response);
         assertNotNull(response.hits());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 3e9cc1f55ff38..17a9c0cd1cfca 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -1297,7 +1297,7 @@ public void testPutDataFrameAnalyticsConfig_GivenRegression() throws Exception {
                 .setLambda(1.0)
                 .setGamma(1.0)
                 .setEta(1.0)
-                .setMaximumNumberTrees(10)
+                .setMaxTrees(10)
                 .setFeatureBagFraction(0.5)
                 .setNumTopFeatureImportanceValues(3)
                 .build())
@@ -1336,11 +1336,13 @@ public void testPutDataFrameAnalyticsConfig_GivenClassification() throws Excepti
                 .setPredictionFieldName("my_dependent_variable_prediction")
                 .setTrainingPercent(80.0)
                 .setRandomizeSeed(42L)
+                .setClassAssignmentObjective(
+                    org.elasticsearch.client.ml.dataframe.Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY)
                 .setNumTopClasses(1)
                 .setLambda(1.0)
                 .setGamma(1.0)
                 .setEta(1.0)
-                .setMaximumNumberTrees(10)
+                .setMaxTrees(10)
                 .setFeatureBagFraction(0.5)
                 .setNumTopFeatureImportanceValues(3)
                 .build())
@@ -2265,7 +2267,7 @@ public void testGetTrainedModelsStats() throws Exception {
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java
index 03911a36884a5..946c707754e04 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
@@ -61,8 +62,8 @@ public void testStringStats() throws IOException {
         assertThat(stats.getDistribution(), hasEntry(equalTo("t"), closeTo(.09, .005)));
     }
 
-    public void testTopMetricsSizeOne() throws IOException {
-        indexTopMetricsData();
+    public void testTopMetricsDoubleMetric() throws IOException {
+        indexTopMetricsDoubleTestData();
         SearchRequest search = new SearchRequest("test");
         search.source().aggregation(new TopMetricsAggregationBuilder(
                 "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v"));
@@ -74,8 +75,34 @@ public void testTopMetricsSizeOne() throws IOException {
         assertThat(metric.getMetrics(), equalTo(singletonMap("v", 3.0)));
     }
 
+    public void testTopMetricsLongMetric() throws IOException {
+        indexTopMetricsLongTestData();
+        SearchRequest search = new SearchRequest("test");
+        search.source().aggregation(new TopMetricsAggregationBuilder(
+                "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v"));
+        SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
+        ParsedTopMetrics top = response.getAggregations().get("test");
+        assertThat(top.getTopMetrics(), hasSize(1));
+        ParsedTopMetrics.TopMetrics metric = top.getTopMetrics().get(0);
+        assertThat(metric.getSort(), equalTo(singletonList(2)));
+        assertThat(metric.getMetrics(), equalTo(singletonMap("v", 3)));
+    }
+
+    public void testTopMetricsDateMetric() throws IOException {
+        indexTopMetricsDateTestData();
+        SearchRequest search = new SearchRequest("test");
+        search.source().aggregation(new TopMetricsAggregationBuilder(
+                "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v"));
+        SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
+        ParsedTopMetrics top = response.getAggregations().get("test");
+        assertThat(top.getTopMetrics(), hasSize(1));
+        ParsedTopMetrics.TopMetrics metric = top.getTopMetrics().get(0);
+        assertThat(metric.getSort(), equalTo(singletonList(2)));
+        assertThat(metric.getMetrics(), equalTo(singletonMap("v", "2020-01-02T01:01:00.000Z")));
+    }
+
     public void testTopMetricsManyMetrics() throws IOException {
-        indexTopMetricsData();
+        indexTopMetricsDoubleTestData();
         SearchRequest search = new SearchRequest("test");
         search.source().aggregation(new TopMetricsAggregationBuilder(
                 "test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v", "m"));
@@ -89,7 +116,7 @@ public void testTopMetricsManyMetrics() throws IOException {
     }
 
     public void testTopMetricsSizeTwo() throws IOException {
-        indexTopMetricsData();
+        indexTopMetricsDoubleTestData();
         SearchRequest search = new SearchRequest("test");
         search.source().aggregation(new TopMetricsAggregationBuilder(
                 "test", new FieldSortBuilder("s").order(SortOrder.DESC), 2, "v"));
@@ -104,10 +131,28 @@ public void testTopMetricsSizeTwo() throws IOException {
         assertThat(metric.getMetrics(), equalTo(singletonMap("v", 2.0)));
     }
 
-    private void indexTopMetricsData() throws IOException {
+    private void indexTopMetricsDoubleTestData() throws IOException {
         BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
         bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", 2.0, "m", 12.0));
         bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", 3.0, "m", 13.0));
         highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
     }
+
+    private void indexTopMetricsLongTestData() throws IOException {
+        BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", 2));
+        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", 3));
+        highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
+    }
+
+    private void indexTopMetricsDateTestData() throws IOException {
+        CreateIndexRequest create = new CreateIndexRequest("test");
+        create.mapping("{\"properties\": {\"v\": {\"type\": \"date\"}}}", XContentType.JSON);
+        highLevelClient().indices().create(create, RequestOptions.DEFAULT);
+        BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", "2020-01-01T01:01:00Z"));
+        bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", "2020-01-02T01:01:00Z"));
+        highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
+    }
+
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
index d1c3a5e657e0f..adf78daacc7d5 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
@@ -139,6 +139,7 @@
 import org.elasticsearch.client.ml.datafeed.DatafeedStats;
 import org.elasticsearch.client.ml.datafeed.DatafeedUpdate;
 import org.elasticsearch.client.ml.datafeed.DelayedDataCheckConfig;
+import org.elasticsearch.client.ml.dataframe.Classification;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDest;
@@ -2969,17 +2970,18 @@ public void testPutDataFrameAnalytics() throws Exception {
             // end::put-data-frame-analytics-outlier-detection-customized
 
             // tag::put-data-frame-analytics-classification
-            DataFrameAnalysis classification = org.elasticsearch.client.ml.dataframe.Classification.builder("my_dependent_variable") // <1>
+            DataFrameAnalysis classification = Classification.builder("my_dependent_variable") // <1>
                 .setLambda(1.0) // <2>
                 .setGamma(5.5) // <3>
                 .setEta(5.5) // <4>
-                .setMaximumNumberTrees(50) // <5>
+                .setMaxTrees(50) // <5>
                 .setFeatureBagFraction(0.4) // <6>
                 .setNumTopFeatureImportanceValues(3) // <7>
                 .setPredictionFieldName("my_prediction_field_name") // <8>
                 .setTrainingPercent(50.0) // <9>
                 .setRandomizeSeed(1234L) // <10>
-                .setNumTopClasses(1) // <11>
+                .setClassAssignmentObjective(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY) // <11>
+                .setNumTopClasses(1) // <12>
                 .build();
             // end::put-data-frame-analytics-classification
 
@@ -2988,7 +2990,7 @@ public void testPutDataFrameAnalytics() throws Exception {
                 .setLambda(1.0) // <2>
                 .setGamma(5.5) // <3>
                 .setEta(5.5) // <4>
-                .setMaximumNumberTrees(50) // <5>
+                .setMaxTrees(50) // <5>
                 .setFeatureBagFraction(0.4) // <6>
                 .setNumTopFeatureImportanceValues(3) // <7>
                 .setPredictionFieldName("my_prediction_field_name") // <8>
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
index 79d78c888880f..0970222c513b9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
@@ -30,12 +30,13 @@ public static Classification randomClassification() {
             .setLambda(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setGamma(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setEta(randomBoolean() ? null : randomDoubleBetween(0.001, 1.0, true))
-            .setMaximumNumberTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
+            .setMaxTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
             .setFeatureBagFraction(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, false))
             .setNumTopFeatureImportanceValues(randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE))
             .setPredictionFieldName(randomBoolean() ? null : randomAlphaOfLength(10))
             .setTrainingPercent(randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true))
             .setRandomizeSeed(randomBoolean() ? null : randomLong())
+            .setClassAssignmentObjective(randomBoolean() ? null : randomFrom(Classification.ClassAssignmentObjective.values()))
             .setNumTopClasses(randomBoolean() ? null : randomIntBetween(0, 10))
             .build();
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java
index eedffb4740d78..1b81f1d0f71af 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/RegressionTests.java
@@ -30,7 +30,7 @@ public static Regression randomRegression() {
             .setLambda(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setGamma(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setEta(randomBoolean() ? null : randomDoubleBetween(0.001, 1.0, true))
-            .setMaximumNumberTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
+            .setMaxTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
             .setFeatureBagFraction(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, false))
             .setNumTopFeatureImportanceValues(randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE))
             .setPredictionFieldName(randomBoolean() ? null : randomAlphaOfLength(10))
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java
index 43ab2e5993fde..1bcefc76434c5 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java
@@ -30,6 +30,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -52,7 +53,11 @@ public static TrainedModelConfig createTestTrainedModelConfig() {
             randomBoolean() ? null : TrainedModelInputTests.createRandomInput(),
             randomBoolean() ? null : randomNonNegativeLong(),
             randomBoolean() ? null : randomNonNegativeLong(),
-            randomBoolean() ? null : randomFrom("platinum", "basic"));
+            randomBoolean() ? null : randomFrom("platinum", "basic"),
+            randomBoolean() ? null :
+                Stream.generate(() -> randomAlphaOfLength(10))
+                    .limit(randomIntBetween(1, 10))
+                    .collect(Collectors.toMap(Function.identity(), (k) -> randomAlphaOfLength(10))));
     }
 
     @Override
diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml
index 245056382f304..08e39ff8dd750 100644
--- a/distribution/docker/docker-compose.yml
+++ b/distribution/docker/docker-compose.yml
@@ -1,5 +1,5 @@
 # Only used for testing the docker images
-version: '3.7'
+version: '3.4'
 services:
   elasticsearch-default-1:
     image: elasticsearch:test
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index ac839e3b9cb95..f7d22f233b840 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -29,7 +29,7 @@ ifeval::["{release-state}"=="unreleased"]
 :rest-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}-SNAPSHOT
 :rest-client-sniffer-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}-SNAPSHOT
 :rest-high-level-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}-SNAPSHOT
-:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}-SNAPSHOT
+:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}-SNAPSHOT
 :painless-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}-SNAPSHOT
 :parent-join-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}-SNAPSHOT
 :percolator-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}-SNAPSHOT
@@ -44,7 +44,7 @@ ifeval::["{release-state}"!="unreleased"]
 :rest-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}
 :rest-client-sniffer-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}
 :rest-high-level-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}
-:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}
+:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}
 :painless-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}
 :parent-join-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}
 :percolator-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}
diff --git a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
index 4be2011340210..cf88d65ae9314 100644
--- a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
+++ b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
@@ -121,7 +121,8 @@ include-tagged::{doc-tests-file}[{api}-classification]
 <8> The name of the prediction field in the results object.
 <9> The percentage of training-eligible rows to be used in training. Defaults to 100%.
 <10> The seed to be used by the random generator that picks which rows are used in training.
-<11> The number of top classes to be reported in the results. Defaults to 2.
+<11> The optimization objective to target when assigning class labels. Defaults to maximize_minimum_recall.
+<12> The number of top classes to be reported in the results. Defaults to 2.
 
 ===== Regression
 
diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc
index 5bb4c3260ac9d..ff6ed7e2b2416 100644
--- a/docs/plugins/integrations.asciidoc
+++ b/docs/plugins/integrations.asciidoc
@@ -145,8 +145,8 @@ releases 2.0 and later do not support rivers.
 * https://micronaut-projects.github.io/micronaut-elasticsearch/latest/guide/index.html[Micronaut Elasticsearch Integration]:
   Integration of Micronaut with Elasticsearch
 
-* https://docs.streampipes.org/docs/user-guide-introduction[StreamPipes]:
-  StreamPipes is a framework that enables users to work with data streams allowing to store data in Elasticsearch.
+* https://streampipes.apache.org[Apache StreamPipes]:
+  StreamPipes is a framework that enables users to work with IoT data streams allowing to store data in Elasticsearch.
 
 * https://metamodel.apache.org/[Apache MetaModel]:
   Providing a common interface for discovery, exploration of metadata and querying of different types of data sources.
diff --git a/docs/python/index.asciidoc b/docs/python/index.asciidoc
index 8def003e7c2ab..61a1570ce585b 100644
--- a/docs/python/index.asciidoc
+++ b/docs/python/index.asciidoc
@@ -5,12 +5,12 @@
 Official low-level client for Elasticsearch. Its goal is to provide common
 ground for all Elasticsearch-related code in Python; because of this it tries
 to be opinion-free and very extendable. The full documentation is available at
-http://elasticsearch-py.rtfd.org/
+http://elasticsearch-py.readthedocs.org/
 
 .Elasticsearch DSL
 ************************************************************************************
 For a more high level client library with more limited scope, have a look at
-http://elasticsearch-dsl.rtfd.org/[elasticsearch-dsl] - a more pythonic library
+http://elasticsearch-dsl.readthedocs.org/[elasticsearch-dsl] - a more pythonic library
 sitting on top of `elasticsearch-py`.
 
 It provides a more convenient and idiomatic way to write and manipulate
diff --git a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
index 8d4652aebe2c5..d6c31d54c9550 100644
--- a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
@@ -286,7 +286,7 @@ precision::     Optional. The string length of the geohashes used to define
                 to precision levels higher than the supported 12 levels,
                 (e.g. for distances <5.6cm) the value is rejected.
 
-bounds:         Optional. The bounding box to filter the points in the bucket.
+bounds::        Optional. The bounding box to filter the points in the bucket.
 
 size::          Optional. The maximum number of geohash buckets to return
                 (defaults to 10,000). When results are trimmed, buckets are
diff --git a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
index cee0b5f7c0f22..5958d09f4ed71 100644
--- a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
@@ -70,23 +70,27 @@ the same sort values then this aggregation could return either document's fields
 
 ==== `metrics`
 
-`metrics` selects the fields to of the "top" document to return. Like most other
-aggregations, `top_metrics` casts these values cast to `double` precision
-floating point numbers. So they have to be numeric. Dates *work*, but they
-come back as a `double` precision floating point containing milliseconds since
-epoch. `keyword` fields aren't allowed.
+`metrics` selects the fields to of the "top" document to return.
 
 You can return multiple metrics by providing a list:
 
 [source,console,id=search-aggregations-metrics-top-metrics-list-of-metrics]
 ----
+PUT /test
+{
+  "mappings": {
+    "properties": {
+      "d": {"type": "date"}
+    }
+  }
+}
 POST /test/_bulk?refresh
 {"index": {}}
-{"s": 1, "v": 3.1415, "m": 1.9}
+{"s": 1, "v": 3.1415, "m": 1, "d": "2020-01-01T00:12:12Z"}
 {"index": {}}
-{"s": 2, "v": 1.0, "m": 6.7}
+{"s": 2, "v": 1.0, "m": 6, "d": "2020-01-02T00:12:12Z"}
 {"index": {}}
-{"s": 3, "v": 2.71828, "m": -12.2}
+{"s": 3, "v": 2.71828, "m": -12, "d": "2019-12-31T00:12:12Z"}
 POST /test/_search?filter_path=aggregations
 {
   "aggs": {
@@ -94,7 +98,8 @@ POST /test/_search?filter_path=aggregations
       "top_metrics": {
         "metrics": [
           {"field": "v"},
-          {"field": "m"}
+          {"field": "m"},
+          {"field": "d"}
         ],
         "sort": {"s": "desc"}
       }
@@ -114,7 +119,8 @@ Which returns:
         "sort": [3],
         "metrics": {
           "v": 2.718280076980591,
-          "m": -12.199999809265137
+          "m": -12,
+          "d": "2019-12-31T00:12:12.000Z"
         }
       } ]
     }
@@ -123,7 +129,6 @@ Which returns:
 ----
 // TESTRESPONSE
 
-
 ==== `size`
 
 `top_metrics` can return the top few document's worth of metrics using the size parameter:
@@ -246,14 +251,14 @@ Which returns:
           "key": "192.168.0.1",
           "doc_count": 2,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2.0 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2 } } ]
           }
         },
         {
           "key": "192.168.0.2",
           "doc_count": 1,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3.0 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3 } } ]
           }
         }
       ],
@@ -303,14 +308,14 @@ Which returns:
           "key": "192.168.0.2",
           "doc_count": 1,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3.0 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3 } } ]
           }
         },
         {
           "key": "192.168.0.1",
           "doc_count": 2,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2.0 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2 } } ]
           }
         }
       ],
diff --git a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
index bcff83c5e9950..d75ce6f003979 100644
--- a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Flatten graph</titleabbrev>
 ++++
 
-experimental[This functionality is marked as experimental in Lucene]
-
 The `flatten_graph` token filter accepts an arbitrary graph token
 stream, such as that produced by
 <<analysis-synonym-graph-tokenfilter>>, and flattens it into a single
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
index 65cdd2575be20..8581d8cb7ec17 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
@@ -4,105 +4,496 @@
 <titleabbrev>Word delimiter graph</titleabbrev>
 ++++
 
-experimental[This functionality is marked as experimental in Lucene]
+Splits tokens at non-alphanumeric characters. The `word_delimiter_graph` filter
+also performs optional token normalization based on a set of rules. By default,
+the filter uses the following rules:
 
-Named `word_delimiter_graph`, it splits words into subwords and performs
-optional transformations on subword groups. Words are split into
-subwords with the following rules:
+* Split tokens at non-alphanumeric characters.
+  The filter uses these characters as delimiters.
+  For example: `Super-Duper` -> `Super`, `Duper`
+* Remove leading or trailing delimiters from each token.
+  For example: `XL---42+'Autocoder'` -> `XL`, `42`, `Autocoder`
+* Split tokens at letter case transitions.
+  For example: `PowerShot` -> `Power`, `Shot`
+* Split tokens at letter-number transitions.
+  For example: `XL500` -> `XL`, `500`
+* Remove the English possessive (`'s`) from the end of each token.
+  For example: `Neil's` -> `Neil`
 
-* split on intra-word delimiters (by default, all non alpha-numeric
-characters).
-* "Wi-Fi" -> "Wi", "Fi"
-* split on case transitions: "PowerShot" -> "Power", "Shot"
-* split on letter-number transitions: "SD500" -> "SD", "500"
-* leading and trailing intra-word delimiters on each subword are
-ignored: "//hello---there, 'dude'" -> "hello", "there", "dude"
-* trailing "'s" are removed for each subword: "O'Neil's" -> "O", "Neil"
+The `word_delimiter_graph` filter uses Lucene's
+{lucene-analysis-docs}/miscellaneous/WordDelimiterGraphFilter.html[WordDelimiterGraphFilter].
 
-Unlike the `word_delimiter`, this token filter correctly handles positions for
-multi terms expansion at search-time when any of the following options
-are set to true:
+[TIP]
+====
+The `word_delimiter_graph` filter was designed to remove punctuation from
+complex identifiers, such as product IDs or part numbers. For these use cases,
+we recommend using the `word_delimiter_graph` filter with the
+<<analysis-keyword-tokenizer,`keyword`>> tokenizer.
 
- * `preserve_original`
- * `catenate_numbers`
- * `catenate_words`
- * `catenate_all`
+Avoid using the `word_delimiter_graph` filter to split hyphenated words, such as
+`wi-fi`. Because users often search for these words both with and without
+hyphens, we recommend using the
+<<analysis-synonym-graph-tokenfilter,`synonym_graph`>> filter instead.
+====
 
-Parameters include:
+[[analysis-word-delimiter-graph-tokenfilter-analyze-ex]]
+==== Example
 
-`generate_word_parts`::
-    If `true` causes parts of words to be
-    generated: "PowerShot" -> "Power" "Shot". Defaults to `true`.
+The following <<indices-analyze,analyze API>> request uses the
+`word_delimiter_graph` filter to split `Neil's-Super-Duper-XL500--42+AutoCoder`
+into normalized tokens using the filter's default rules:
 
-`generate_number_parts`::
-    If `true` causes number subwords to be
-    generated: "500-42" -> "500" "42". Defaults to `true`.
+[source,console]
+----
+GET /_analyze
+{
+  "tokenizer": "keyword",
+  "filter": [ "word_delimiter_graph" ],
+  "text": "Neil's-Super-Duper-XL500--42+AutoCoder"
+}
+----
 
-`catenate_words`::
-    If `true` causes maximum runs of word parts to be
-    catenated: "wi-fi" -> "wifi". Defaults to `false`.
+The filter produces the following tokens:
 
-`catenate_numbers`::
-    If `true` causes maximum runs of number parts to
-    be catenated: "500-42" -> "50042". Defaults to `false`.
+[source,txt]
+----
+[ Neil, Super, Duper, XL, 500, 42, Auto, Coder ]
+----
+
+////
+[source,console-result]
+----
+{
+  "tokens": [
+    {
+      "token": "Neil",
+      "start_offset": 0,
+      "end_offset": 4,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "Super",
+      "start_offset": 7,
+      "end_offset": 12,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "Duper",
+      "start_offset": 13,
+      "end_offset": 18,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "XL",
+      "start_offset": 19,
+      "end_offset": 21,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "500",
+      "start_offset": 21,
+      "end_offset": 24,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "42",
+      "start_offset": 26,
+      "end_offset": 28,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "Auto",
+      "start_offset": 29,
+      "end_offset": 33,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "Coder",
+      "start_offset": 33,
+      "end_offset": 38,
+      "type": "word",
+      "position": 7
+    }
+  ]
+}
+----
+////
+
+[[analysis-word-delimiter-graph-tokenfilter-analyzer-ex]]
+==== Add to an analyzer
+
+The following <<indices-create-index,create index API>> request uses the
+`word_delimiter_graph` filter to configure a new
+<<analysis-custom-analyzer,custom analyzer>>.
+
+[source,console]
+----
+PUT /my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "filter": [ "word_delimiter_graph" ]
+        }
+      }
+    }
+  }
+}
+----
+
+[WARNING]
+====
+Avoid using the `word_delimiter_graph` filter with tokenizers that remove
+punctuation, such as the <<analysis-standard-tokenizer,`standard`>> tokenizer.
+This could prevent the `word_delimiter_graph` filter from splitting tokens
+correctly. It can also interfere with the filter's configurable parameters, such
+as <<word-delimiter-graph-tokenfilter-catenate-all,`catenate_all`>> or
+<<word-delimiter-graph-tokenfilter-preserve-original,`preserve_original`>>. We
+recommend using the <<analysis-keyword-tokenizer,`keyword`>> or
+<<analysis-whitespace-tokenizer,`whitespace`>> tokenizer instead.
+====
+
+[[word-delimiter-graph-tokenfilter-configure-parms]]
+==== Configurable parameters
 
+[[word-delimiter-graph-tokenfilter-adjust-offsets]]
+`adjust_offsets`::
++
+--
+(Optional, boolean)
+If `true`, the filter adjusts the offsets of split or catenated tokens to better
+reflect their actual position in the token stream. Defaults to `true`.
+
+[WARNING]
+====
+Set `adjust_offsets` to `false` if your analyzer uses filters, such as the
+<<analysis-trim-tokenfilter,`trim`>> filter, that change the length of tokens
+without changing their offsets. Otherwise, the `word_delimiter_graph` filter
+could produce tokens with illegal offsets.
+====
+--
+
+[[word-delimiter-graph-tokenfilter-catenate-all]]
 `catenate_all`::
-    If `true` causes all subword parts to be catenated:
-    "wi-fi-4000" -> "wifi4000". Defaults to `false`.
++
+--
+(Optional, boolean)
+If `true`, the filter produces catenated tokens for chains of alphanumeric
+characters separated by non-alphabetic delimiters. For example:
+`super-duper-xl-500` -> [ **`superduperxl500`**, `super`, `duper`, `xl`, `500` ].
+Defaults to `false`.
 
-`split_on_case_change`::
-    If `true` causes "PowerShot" to be two tokens;
-    ("Power-Shot" remains two parts regards). Defaults to `true`.
+[WARNING]
+====
+Setting this parameter to `true` produces multi-position tokens, which are not
+supported by indexing.
+
+If this parameter is `true`, avoid using this filter in an index analyzer or
+use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
+this filter to make the token stream suitable for indexing.
+
+When used for search analysis, catenated tokens can cause problems for the
+<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
+rely on token position for matching. Avoid setting this parameter to `true` if
+you plan to use these queries.
+====
+--
+
+[[word-delimiter-graph-tokenfilter-catenate-numbers]]
+`catenate_numbers`::
++
+--
+(Optional, boolean)
+If `true`, the filter produces catenated tokens for chains of numeric characters
+separated by non-alphabetic delimiters. For example: `01-02-03` ->
+[ **`010203`**, `01`, `02`, `03` ]. Defaults to `false`.
+
+[WARNING]
+====
+Setting this parameter to `true` produces multi-position tokens, which are not
+supported by indexing.
+
+If this parameter is `true`, avoid using this filter in an index analyzer or
+use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
+this filter to make the token stream suitable for indexing.
 
+When used for search analysis, catenated tokens can cause problems for the
+<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
+rely on token position for matching. Avoid setting this parameter to `true` if
+you plan to use these queries.
+====
+--
+
+[[word-delimiter-graph-tokenfilter-catenate-words]]
+`catenate_words`::
++
+--
+(Optional, boolean)
+If `true`, the filter produces catenated tokens for chains of alphabetical
+characters separated by non-alphabetic delimiters. For example: `super-duper-xl`
+-> [ **`superduperxl`**, `super`, `duper`, `xl` ]. Defaults to `false`.
+
+[WARNING]
+====
+Setting this parameter to `true` produces multi-position tokens, which are not
+supported by indexing.
+
+If this parameter is `true`, avoid using this filter in an index analyzer or
+use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
+this filter to make the token stream suitable for indexing.
+
+When used for search analysis, catenated tokens can cause problems for the
+<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
+rely on token position for matching. Avoid setting this parameter to `true` if
+you plan to use these queries.
+====
+--
+
+`generate_number_parts`::
+(Optional, boolean)
+If `true`, the filter includes tokens consisting of only numeric characters in
+the output. If `false`, the filter excludes these tokens from the output.
+Defaults to `true`.
+
+`generate_word_parts`::
+(Optional, boolean)
+If `true`, the filter includes tokens consisting of only alphabetical characters
+in the output. If `false`, the filter excludes these tokens from the output.
+Defaults to `true`.
+
+[[word-delimiter-graph-tokenfilter-preserve-original]]
 `preserve_original`::
-    If `true` includes original words in subwords:
-    "500-42" -> "500-42" "500" "42". Defaults to `false`.
++
+--
+(Optional, boolean)
+If `true`, the filter includes the original version of any split tokens in the
+output. This original version includes non-alphanumeric delimiters. For example:
+`super-duper-xl-500` -> [ **`super-duper-xl-500`**, `super`, `duper`, `xl`,
+`500` ]. Defaults to `false`.
+
+[WARNING]
+====
+Setting this parameter to `true` produces multi-position tokens, which are not
+supported by indexing.
+
+If this parameter is `true`, avoid using this filter in an index analyzer or
+use the <<analysis-flatten-graph-tokenfilter,`flatten_graph`>> filter after
+this filter to make the token stream suitable for indexing.
+====
+--
+
+`protected_words`::
+(Optional, array of strings)
+Array of tokens the filter won't split.
+
+`protected_words_path`::
++
+--
+(Optional, string)
+Path to a file that contains a list of tokens the filter won't split.
+
+This path must be absolute or relative to the `config` location, and the file
+must be UTF-8 encoded. Each token in the file must be separated by a line
+break.
+--
+
+`split_on_case_change`::
+(Optional, boolean)
+If `true`, the filter splits tokens at letter case transitions. For example:
+`camelCase` -> [ `camel`, `Case` ]. Defaults to `true`.
 
 `split_on_numerics`::
-    If `true` causes "j2se" to be three tokens; "j"
-    "2" "se". Defaults to `true`.
+(Optional, boolean)
+If `true`, the filter splits tokens at letter-number transitions. For example:
+`j2se` -> [ `j`, `2`, `se` ]. Defaults to `true`.
 
 `stem_english_possessive`::
-    If `true` causes trailing "'s" to be
-    removed for each subword: "O'Neil's" -> "O", "Neil". Defaults to `true`.
+(Optional, boolean)
+If `true`, the filter removes the English possessive (`'s`) from the end of each
+token. For example: `O'Neil's` -> [ `O`, `Neil` ]. Defaults to `true`.
 
-Advance settings include:
+`type_table`::
++
+--
+(Optional, array of strings)
+Array of custom type mappings for characters. This allows you to map
+non-alphanumeric characters as numeric or alphanumeric to avoid splitting on
+those characters.
 
-`protected_words`::
-    A list of protected words from being delimiter.
-    Either an array, or also can set `protected_words_path` which resolved
-    to a file configured with protected words (one on each line).
-    Automatically resolves to `config/` based location if exists.
+For example, the following array maps the plus (`+`) and hyphen (`-`) characters
+as alphanumeric, which means they won't be treated as delimiters:
 
-`adjust_offsets`::
-    By default, the filter tries to output subtokens with adjusted offsets
-    to reflect their actual position in the token stream.  However, when
-    used in combination with other filters that alter the length or starting
-    position of tokens without changing their offsets
-    (e.g. <<analysis-trim-tokenfilter,`trim`>>) this can cause tokens with
-    illegal offsets to be emitted.  Setting `adjust_offsets` to false will
-    stop `word_delimiter_graph` from adjusting these internal offsets.
+`[ "+ => ALPHA", "- => ALPHA" ]`
 
-`type_table`::
-    A custom type mapping table, for example (when configured
-    using `type_table_path`):
-
-[source,type_table]
---------------------------------------------------
-    # Map the $, %, '.', and ',' characters to DIGIT
-    # This might be useful for financial data.
-    $ => DIGIT
-    % => DIGIT
-    . => DIGIT
-    \\u002C => DIGIT
-
-    # in some cases you might not want to split on ZWJ
-    # this also tests the case where we need a bigger byte[]
-    # see http://en.wikipedia.org/wiki/Zero-width_joiner
-    \\u200D => ALPHANUM
---------------------------------------------------
-
-NOTE: Using a tokenizer like the `standard` tokenizer may interfere with
-the `catenate_*` and `preserve_original` parameters, as the original
-string may already have lost punctuation during tokenization.  Instead,
-you may want to use the `whitespace` tokenizer.
+Supported types include:
+
+* `ALPHA` (Alphabetical)
+* `ALPHANUM` (Alphanumeric)
+* `DIGIT` (Numeric)
+* `LOWER` (Lowercase alphabetical)
+* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
+* `UPPER` (Uppercase alphabetical)
+--
+
+`type_table_path`::
++
+--
+(Optional, string)
+Path to a file that contains custom type mappings for characters. This allows
+you to map non-alphanumeric characters as numeric or alphanumeric to avoid
+splitting on those characters.
+
+For example, the contents of this file may contain the following:
+
+[source,txt]
+----
+# Map the $, %, '.', and ',' characters to DIGIT
+# This might be useful for financial data.
+$ => DIGIT
+% => DIGIT
+. => DIGIT
+\\u002C => DIGIT
+
+# in some cases you might not want to split on ZWJ
+# this also tests the case where we need a bigger byte[]
+# see http://en.wikipedia.org/wiki/Zero-width_joiner
+\\u200D => ALPHANUM
+----
+
+Supported types include:
+
+* `ALPHA` (Alphabetical)
+* `ALPHANUM` (Alphanumeric)
+* `DIGIT` (Numeric)
+* `LOWER` (Lowercase alphabetical)
+* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
+* `UPPER` (Uppercase alphabetical)
+
+This file path must be absolute or relative to the `config` location, and the
+file must be UTF-8 encoded. Each mapping in the file must be separated by a line
+break.
+--
+
+[[analysis-word-delimiter-graph-tokenfilter-customize]]
+==== Customize
+
+To customize the `word_delimiter_graph` filter, duplicate it to create the basis
+for a new custom token filter. You can modify the filter using its configurable
+parameters.
+
+For example, the following request creates a `word_delimiter_graph`
+filter that uses the following rules:
+
+* Split tokens at non-alphanumeric characters, _except_ the hyphen (`-`)
+  character.
+* Remove leading or trailing delimiters from each token.
+* Do _not_ split tokens at letter case transitions.
+* Do _not_ split tokens at letter-number transitions.
+* Remove the English possessive (`'s`) from the end of each token.
+
+[source,console]
+----
+PUT /my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "filter": [ "my_custom_word_delimiter_graph_filter" ]
+        }
+      },
+      "filter": {
+        "my_custom_word_delimiter_graph_filter": {
+          "type": "word_delimiter_graph",
+          "type_table": [ "- => ALPHA" ],
+          "split_on_case_change": false,
+          "split_on_numerics": false,
+          "stem_english_possessive": true
+        }
+      }
+    }
+  }
+}
+----
+
+[[analysis-word-delimiter-graph-differences]]
+==== Differences between `word_delimiter_graph` and `word_delimiter` 
+
+Both the  `word_delimiter_graph` and
+<<analysis-word-delimiter-tokenfilter,`word_delimiter`>> filters produce tokens
+that span multiple positions when any of the following parameters are `true`:
+
+ * <<word-delimiter-graph-tokenfilter-catenate-all,`catenate_all`>>
+ * <<word-delimiter-graph-tokenfilter-catenate-numbers,`catenate_numbers`>>
+ * <<word-delimiter-graph-tokenfilter-catenate-words,`catenate_words`>>
+ * <<word-delimiter-graph-tokenfilter-preserve-original,`preserve_original`>>
+
+However, only the `word_delimiter_graph` filter assigns multi-position tokens a
+`positionLength` attribute, which indicates the number of positions a token
+spans. This ensures the `word_delimiter_graph` filter always produces valid token
+https://en.wikipedia.org/wiki/Directed_acyclic_graph[graphs].
+
+The `word_delimiter` filter does not assign multi-position tokens a
+`positionLength` attribute. This means it produces invalid graphs for streams
+including these tokens.
+
+While indexing does not support token graphs containing multi-position tokens,
+queries, such as the <<query-dsl-match-query-phrase,`match_phrase`>> query, can
+use these graphs to generate multiple sub-queries from a single query string.
+
+To see how token graphs produced by the `word_delimiter` and
+`word_delimiter_graph` filters differ, check out the following example.
+
+.*Example*
+[%collapsible]
+====
+
+[[analysis-word-delimiter-graph-basic-token-graph]]
+*Basic token graph*
+
+Both the `word_delimiter` and `word_delimiter_graph` produce the following token
+graph for `PowerShot2000` when the following parameters are `false`:
+
+ * <<word-delimiter-graph-tokenfilter-catenate-all,`catenate_all`>>
+ * <<word-delimiter-graph-tokenfilter-catenate-numbers,`catenate_numbers`>>
+ * <<word-delimiter-graph-tokenfilter-catenate-words,`catenate_words`>>
+ * <<word-delimiter-graph-tokenfilter-preserve-original,`preserve_original`>>
+
+This graph does not contain multi-position tokens. All tokens span only one
+position.
+
+image::images/analysis/token-graph-basic.svg[align="center"]
+
+[[analysis-word-delimiter-graph-wdg-token-graph]]
+*`word_delimiter_graph` graph with a multi-position token*
+
+The `word_delimiter_graph` filter produces the following token graph for
+`PowerShot2000` when `catenate_words` is `true`.
+
+This graph correctly indicates the catenated `PowerShot` token spans two
+positions.
+
+image::images/analysis/token-graph-wdg.svg[align="center"]
+
+[[analysis-word-delimiter-graph-wd-token-graph]]
+*`word_delimiter` graph with a multi-position token*
+
+When `catenate_words` is `true`, the `word_delimiter` filter produces
+the following token graph for `PowerShot2000`.
+
+Note that the catenated `PowerShot` token should span two positions but only
+spans one in the token graph, making it invalid.
+
+image::images/analysis/token-graph-wd.svg[align="center"]
+
+====
\ No newline at end of file
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
index 25074b2725ea4..02d6257cb4ee9 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
@@ -4,84 +4,379 @@
 <titleabbrev>Word delimiter</titleabbrev>
 ++++
 
-Named `word_delimiter`, it Splits words into subwords and performs
-optional transformations on subword groups. Words are split into
-subwords with the following rules:
+[WARNING]
+====
+We recommend using the
+<<analysis-word-delimiter-graph-tokenfilter,`word_delimiter_graph`>> instead of
+the `word_delimiter` filter.
 
-* split on intra-word delimiters (by default, all non alpha-numeric
-characters): "Wi-Fi" -> "Wi", "Fi"
-* split on case transitions: "PowerShot" -> "Power", "Shot"
-* split on letter-number transitions: "SD500" -> "SD", "500"
-* leading and trailing intra-word delimiters on each subword are
-ignored: "//hello---there, 'dude'" -> "hello", "there", "dude"
-* trailing "'s" are removed for each subword: "O'Neil's" -> "O", "Neil"
+The `word_delimiter` filter can produce invalid token graphs. See
+<<analysis-word-delimiter-graph-differences>>.
 
-Parameters include:
+The `word_delimiter` filter also uses Lucene's
+{lucene-analysis-docs}/miscellaneous/WordDelimiterFilter.html[WordDelimiterFilter],
+which is marked as deprecated. 
+====
 
-`generate_word_parts`::
-    If `true` causes parts of words to be
-    generated: "Power-Shot", "(Power,Shot)" -> "Power" "Shot". Defaults to `true`.
+Splits tokens at non-alphanumeric characters. The `word_delimiter` filter
+also performs optional token normalization based on a set of rules. By default,
+the filter uses the following rules:
 
-`generate_number_parts`::
-    If `true` causes number subwords to be
-    generated: "500-42" -> "500" "42". Defaults to `true`.
+* Split tokens at non-alphanumeric characters.
+  The filter uses these characters as delimiters.
+  For example: `Super-Duper` -> `Super`, `Duper`
+* Remove leading or trailing delimiters from each token.
+  For example: `XL---42+'Autocoder'` -> `XL`, `42`, `Autocoder`
+* Split tokens at letter case transitions.
+  For example: `PowerShot` -> `Power`, `Shot`
+* Split tokens at letter-number transitions.
+  For example: `XL500` -> `XL`, `500`
+* Remove the English possessive (`'s`) from the end of each token.
+  For example: `Neil's` -> `Neil`
 
-`catenate_words`::
-    If `true` causes maximum runs of word parts to be
-    catenated: "wi-fi" -> "wifi". Defaults to `false`.
+[TIP]
+====
+The `word_delimiter` filter was designed to remove punctuation from complex
+identifiers, such as product IDs or part numbers. For these use cases, we
+recommend using the `word_delimiter` filter with the
+<<analysis-keyword-tokenizer,`keyword`>> tokenizer.
 
-`catenate_numbers`::
-    If `true` causes maximum runs of number parts to
-    be catenated: "500-42" -> "50042". Defaults to `false`.
+Avoid using the `word_delimiter` filter to split hyphenated words, such as
+`wi-fi`. Because users often search for these words both with and without
+hyphens, we recommend using the
+<<analysis-synonym-graph-tokenfilter,`synonym_graph`>> filter instead.
+====
+
+[[analysis-word-delimiter-tokenfilter-analyze-ex]]
+==== Example
+
+The following <<indices-analyze,analyze API>> request uses the
+`word_delimiter` filter to split `Neil's-Super-Duper-XL500--42+AutoCoder`
+into normalized tokens using the filter's default rules:
+
+[source,console]
+----
+GET /_analyze
+{
+  "tokenizer": "keyword",
+  "filter": [ "word_delimiter" ],
+  "text": "Neil's-Super-Duper-XL500--42+AutoCoder"
+}
+----
+
+The filter produces the following tokens:
+
+[source,txt]
+----
+[ Neil, Super, Duper, XL, 500, 42, Auto, Coder ]
+----
+
+////
+[source,console-result]
+----
+{
+  "tokens": [
+    {
+      "token": "Neil",
+      "start_offset": 0,
+      "end_offset": 4,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "Super",
+      "start_offset": 7,
+      "end_offset": 12,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "Duper",
+      "start_offset": 13,
+      "end_offset": 18,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "XL",
+      "start_offset": 19,
+      "end_offset": 21,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "500",
+      "start_offset": 21,
+      "end_offset": 24,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "42",
+      "start_offset": 26,
+      "end_offset": 28,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "Auto",
+      "start_offset": 29,
+      "end_offset": 33,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "Coder",
+      "start_offset": 33,
+      "end_offset": 38,
+      "type": "word",
+      "position": 7
+    }
+  ]
+}
+----
+////
+
+[analysis-word-delimiter-tokenfilter-analyzer-ex]]
+==== Add to an analyzer
+
+The following <<indices-create-index,create index API>> request uses the
+`word_delimiter` filter to configure a new
+<<analysis-custom-analyzer,custom analyzer>>.
+
+[source,console]
+----
+PUT /my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "filter": [ "word_delimiter" ]
+        }
+      }
+    }
+  }
+}
+----
+
+[WARNING]
+====
+Avoid using the `word_delimiter` filter with tokenizers that remove punctuation,
+such as the <<analysis-standard-tokenizer,`standard`>> tokenizer. This could
+prevent the `word_delimiter` filter from splitting tokens correctly. It can also
+interfere with the filter's configurable parameters, such as `catenate_all` or
+`preserve_original`. We recommend using the
+<<analysis-keyword-tokenizer,`keyword`>> or
+<<analysis-whitespace-tokenizer,`whitespace`>> tokenizer instead.
+====
+
+[[word-delimiter-tokenfilter-configure-parms]]
+==== Configurable parameters
 
 `catenate_all`::
-    If `true` causes all subword parts to be catenated:
-    "wi-fi-4000" -> "wifi4000". Defaults to `false`.
++
+--
+(Optional, boolean)
+If `true`, the filter produces catenated tokens for chains of alphanumeric
+characters separated by non-alphabetic delimiters. For example:
+`super-duper-xl-500` -> [ `super`, **`superduperxl500`**, `duper`, `xl`, `500`
+]. Defaults to `false`.
 
-`split_on_case_change`::
-    If `true` causes "PowerShot" to be two tokens;
-    ("Power-Shot" remains two parts regards). Defaults to `true`.
+[WARNING]
+====
+When used for search analysis, catenated tokens can cause problems for the
+<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
+rely on token position for matching. Avoid setting this parameter to `true` if
+you plan to use these queries.
+====
+--
+
+`catenate_numbers`::
++
+--
+(Optional, boolean)
+If `true`, the filter produces catenated tokens for chains of numeric characters
+separated by non-alphabetic delimiters. For example: `01-02-03` ->
+[ `01`, **`010203`**, `02`, `03` ]. Defaults to `false`.
+
+[WARNING]
+====
+When used for search analysis, catenated tokens can cause problems for the
+<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
+rely on token position for matching. Avoid setting this parameter to `true` if
+you plan to use these queries.
+====
+--
+
+`catenate_words`::
++
+--
+(Optional, boolean)
+If `true`, the filter produces catenated tokens for chains of alphabetical
+characters separated by non-alphabetic delimiters. For example: `super-duper-xl`
+-> [ `super`, **`superduperxl`**, `duper`, `xl` ]. Defaults to `false`.
+
+[WARNING]
+====
+When used for search analysis, catenated tokens can cause problems for the
+<<query-dsl-match-query-phrase,`match_phrase`>> query and other queries that
+rely on token position for matching. Avoid setting this parameter to `true` if
+you plan to use these queries.
+====
+--
+
+`generate_number_parts`::
+(Optional, boolean)
+If `true`, the filter includes tokens consisting of only numeric characters in
+the output. If `false`, the filter excludes these tokens from the output.
+Defaults to `true`.
+
+`generate_word_parts`::
+(Optional, boolean)
+If `true`, the filter includes tokens consisting of only alphabetical characters
+in the output. If `false`, the filter excludes these tokens from the output.
+Defaults to `true`.
 
 `preserve_original`::
-    If `true` includes original words in subwords:
-    "500-42" -> "500-42" "500" "42". Defaults to `false`.
+(Optional, boolean)
+If `true`, the filter includes the original version of any split tokens in the
+output. This original version includes non-alphanumeric delimiters. For example:
+`super-duper-xl-500` -> [ **`super-duper-xl-500`**, `super`, `duper`, `xl`,
+`500` ]. Defaults to `false`.
+
+`protected_words`::
+(Optional, array of strings)
+Array of tokens the filter won't split.
+
+`protected_words_path`::
++
+--
+(Optional, string)
+Path to a file that contains a list of tokens the filter won't split.
+
+This path must be absolute or relative to the `config` location, and the file
+must be UTF-8 encoded. Each token in the file must be separated by a line
+break.
+--
+
+`split_on_case_change`::
+(Optional, boolean)
+If `true`, the filter splits tokens at letter case transitions. For example:
+`camelCase` -> [ `camel`, `Case` ]. Defaults to `true`.
 
 `split_on_numerics`::
-    If `true` causes "j2se" to be three tokens; "j"
-    "2" "se". Defaults to `true`.
+(Optional, boolean)
+If `true`, the filter splits tokens at letter-number transitions. For example:
+`j2se` -> [ `j`, `2`, `se` ]. Defaults to `true`.
 
 `stem_english_possessive`::
-    If `true` causes trailing "'s" to be
-    removed for each subword: "O'Neil's" -> "O", "Neil". Defaults to `true`.
+(Optional, boolean)
+If `true`, the filter removes the English possessive (`'s`) from the end of each
+token. For example: `O'Neil's` -> [ `O`, `Neil` ]. Defaults to `true`.
 
-Advance settings include:
+`type_table`::
++
+--
+(Optional, array of strings)
+Array of custom type mappings for characters. This allows you to map
+non-alphanumeric characters as numeric or alphanumeric to avoid splitting on
+those characters.
 
-`protected_words`::
-    A list of protected words from being delimiter.
-    Either an array, or also can set `protected_words_path` which resolved
-    to a file configured with protected words (one on each line).
-    Automatically resolves to `config/` based location if exists.
+For example, the following array maps the plus (`+`) and hyphen (`-`) characters
+as alphanumeric, which means they won't be treated as delimiters:
 
-`type_table`::
-    A custom type mapping table, for example (when configured
-    using `type_table_path`):
-
-[source,type_table]
---------------------------------------------------
-    # Map the $, %, '.', and ',' characters to DIGIT
-    # This might be useful for financial data.
-    $ => DIGIT
-    % => DIGIT
-    . => DIGIT
-    \\u002C => DIGIT
-
-    # in some cases you might not want to split on ZWJ
-    # this also tests the case where we need a bigger byte[]
-    # see http://en.wikipedia.org/wiki/Zero-width_joiner
-    \\u200D => ALPHANUM
---------------------------------------------------
-
-NOTE: Using a tokenizer like the `standard` tokenizer may interfere with
-the `catenate_*` and `preserve_original` parameters, as the original
-string may already have lost punctuation during tokenization.  Instead,
-you may want to use the `whitespace` tokenizer.
+`[ "+ => ALPHA", "- => ALPHA" ]`
+
+Supported types include:
+
+* `ALPHA` (Alphabetical)
+* `ALPHANUM` (Alphanumeric)
+* `DIGIT` (Numeric)
+* `LOWER` (Lowercase alphabetical)
+* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
+* `UPPER` (Uppercase alphabetical)
+--
+
+`type_table_path`::
++
+--
+(Optional, string)
+Path to a file that contains custom type mappings for characters. This allows
+you to map non-alphanumeric characters as numeric or alphanumeric to avoid
+splitting on those characters.
+
+For example, the contents of this file may contain the following:
+
+[source,txt]
+----
+# Map the $, %, '.', and ',' characters to DIGIT
+# This might be useful for financial data.
+$ => DIGIT
+% => DIGIT
+. => DIGIT
+\\u002C => DIGIT
+
+# in some cases you might not want to split on ZWJ
+# this also tests the case where we need a bigger byte[]
+# see http://en.wikipedia.org/wiki/Zero-width_joiner
+\\u200D => ALPHANUM
+----
+
+Supported types include:
+
+* `ALPHA` (Alphabetical)
+* `ALPHANUM` (Alphanumeric)
+* `DIGIT` (Numeric)
+* `LOWER` (Lowercase alphabetical)
+* `SUBWORD_DELIM` (Non-alphanumeric delimiter)
+* `UPPER` (Uppercase alphabetical)
+
+This file path must be absolute or relative to the `config` location, and the
+file must be UTF-8 encoded. Each mapping in the file must be separated by a line
+break.
+--
+
+[[analysis-word-delimiter-tokenfilter-customize]]
+==== Customize
+
+To customize the `word_delimiter` filter, duplicate it to create the basis
+for a new custom token filter. You can modify the filter using its configurable
+parameters.
+
+For example, the following request creates a `word_delimiter`
+filter that uses the following rules:
+
+* Split tokens at non-alphanumeric characters, _except_ the hyphen (`-`)
+  character.
+* Remove leading or trailing delimiters from each token.
+* Do _not_ split tokens at letter case transitions.
+* Do _not_ split tokens at letter-number transitions.
+* Remove the English possessive (`'s`) from the end of each token.
+
+[source,console]
+----
+PUT /my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "filter": [ "my_custom_word_delimiter_filter" ]
+        }
+      },
+      "filter": {
+        "my_custom_word_delimiter_filter": {
+          "type": "word_delimiter",
+          "type_table": [ "- => ALPHA" ],
+          "split_on_case_change": false,
+          "split_on_numerics": false,
+          "stem_english_possessive": true
+        }
+      }
+    }
+  }
+}
+----
\ No newline at end of file
diff --git a/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
index 5da001640a027..67c0cefc98957 100644
--- a/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
@@ -1,8 +1,6 @@
 [[analysis-simplepattern-tokenizer]]
 === Simple Pattern Tokenizer
 
-experimental[This functionality is marked as experimental in Lucene]
-
 The `simple_pattern` tokenizer uses a regular expression to capture matching
 text as terms. The set of regular expression features it supports is more
 limited than the <<analysis-pattern-tokenizer,`pattern`>> tokenizer, but the
diff --git a/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
index 55be14c45638a..3f24233334e57 100644
--- a/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
@@ -1,8 +1,6 @@
 [[analysis-simplepatternsplit-tokenizer]]
 === Simple Pattern Split Tokenizer
 
-experimental[This functionality is marked as experimental in Lucene]
-
 The `simple_pattern_split` tokenizer uses a regular expression to split the
 input into terms at pattern matches. The set of regular expression features it
 supports is more limited than the <<analysis-pattern-tokenizer,`pattern`>>
diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc
index d7eff24e576d8..05931ea2d6a16 100644
--- a/docs/reference/cluster/remote-info.asciidoc
+++ b/docs/reference/cluster/remote-info.asciidoc
@@ -17,25 +17,20 @@ Returns configured remote cluster information.
 ==== {api-description-title}
 
 The cluster remote info API allows you to retrieve all of the configured
-remote cluster information. It returns connection and endpoint information keyed 
+remote cluster information. It returns connection and endpoint information keyed
 by the configured remote cluster alias.
 
 
 [[cluster-remote-info-api-response-body]]
 ==== {api-response-body-title}
 
-`seeds`::
-	The configured initial seed transport addresses of the remote cluster.
+`mode`::
+    Connection mode for the remote cluster. Returned values are `sniff` and
+    `proxy`.
 
 `connected`::
 	True if there is at least one connection to the remote cluster.
 
-`num_nodes_connected`::
-    The number of connected nodes in the remote cluster.
-
-`max_connections_per_cluster`::
-	The maximum number of connections maintained for the remote cluster.
-
 `initial_connect_timeout`::
 	The initial connect timeout for remote cluster connections.
 
@@ -43,3 +38,26 @@ by the configured remote cluster alias.
 `skip_unavailable`::
     Whether the remote cluster is skipped in case it is searched through
     a {ccs} request but none of its nodes are available.
+
+`seeds`::
+    Initial seed transport addresses of the remote cluster when sniff mode is
+    configured.
+
+`num_nodes_connected`::
+    Number of connected nodes in the remote cluster when sniff mode is
+    configured.
+
+`max_connections_per_cluster`::
+    Maximum number of connections maintained for the remote cluster when sniff
+    mode is configured.
+
+`proxy_address`::
+    Address for remote connections when proxy mode is configured.
+
+`num_proxy_sockets_connected`::
+    Number of open socket connections to the remote cluster when proxy mode
+    is configured.
+
+`max_proxy_socket_connections`::
+    The maximum number of socket connections to the remote cluster when proxy
+    mode is configured.
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index cc9b41f8b96df..9ae24b17eaae1 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -18,9 +18,9 @@ Returns cluster statistics.
 [[cluster-stats-api-desc]]
 ==== {api-description-title}
 
-The Cluster Stats API allows to retrieve statistics from a cluster wide 
-perspective. The API returns basic index metrics (shard numbers, store size, 
-memory usage) and information about the current nodes that form the cluster 
+The Cluster Stats API allows to retrieve statistics from a cluster wide
+perspective. The API returns basic index metrics (shard numbers, store size,
+memory usage) and information about the current nodes that form the cluster
 (number, roles, os, jvm versions, memory usage, cpu and installed plugins).
 
 
@@ -72,6 +72,16 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=cluster-health-status]
 +
 See <<cluster-health>>.
 
+[NOTE]
+====
+The remaining statistics are grouped by section.
+====
+
+[[cluster-stats-api-response-body-indices]]
+===== `indices` section
+
+[%collapsible]
+====
 `indices.count`::
 (integer)
 Total number of indices with shards assigned to selected nodes.
@@ -338,7 +348,290 @@ This object is not populated by the cluster stats API.
 +
 To get information on segment files, use the <<cluster-nodes-stats,node stats
 API>>.
+====
+
+[[cluster-stats-api-response-body-nodes]]
+===== `nodes` section
+
+[%collapsible]
+====
+`nodes.count.total`::
+(integer)
+Total number of nodes selected by the request's <<cluster-nodes,node filters>>.
+
+`nodes.count.coordinating_only`::
+(integer)
+Number of selected nodes without a <<modules-node,role>>. These nodes are
+considered <<coordinating-only-node,coordinating only>> nodes.
+
+`nodes.count.<role>`::
+(integer)
+Number of selected nodes with the role. For a list of roles, see
+<<modules-node>>.
+
+`nodes.versions`::
+(array of strings)
+Array of {es} versions used on selected nodes.
+
+`nodes.os.available_processors`::
+(integer)
+Number of processors available to JVM across all selected nodes.
+
+`nodes.os.allocated_processors`::
+(integer)
+Number of processors used to calculate thread pool size across all selected
+nodes.
++
+This number can be set with the `processors` setting of a node and defaults to
+the number of processors reported by the OS. In both cases, this number will
+never be larger than `32`.
+
+`nodes.os.names`::
+(array of objects)
+Contains details about operating systems used by one or more selected nodes.
+
+`nodes.os.names.name`:::
+(string)
+Name of an operating system used by one or more selected nodes.
+
+`nodes.os.names.count`:::
+(string)
+Number of selected nodes using the operating system.
+
+`nodes.os.pretty_names`::
+(array of objects)
+Contains details about operating systems used by one or more selected nodes.
+
+`nodes.os.pretty_names.pretty_name`:::
+(string)
+Human-readable name of an operating system used by one or more selected nodes.
+
+`nodes.os.pretty_names.count`:::
+(string)
+Number of selected nodes using the operating system.
+
+`nodes.os.mem.total`::
+(<<byte-units, byte units>>)
+Total amount of physical memory across all selected nodes.
+
+`nodes.os.mem.total_in_bytes`::
+(integer)
+Total amount, in bytes, of physical memory across all selected nodes.
+
+`nodes.os.mem.free`::
+(<<byte-units, byte units>>)
+Amount of free physical memory across all selected nodes.
+
+`nodes.os.mem.free_in_bytes`::
+(integer)
+Amount, in bytes, of free physical memory across all selected nodes.
+
+`nodes.os.mem.used`::
+(<<byte-units, byte units>>)
+Amount of physical memory in use across all selected nodes.
+
+`nodes.os.mem.used_in_bytes`::
+(integer)
+Amount, in bytes, of physical memory in use across all selected nodes.
+
+`nodes.os.mem.free_percent`::
+(integer)
+Percentage of free physical memory across all selected nodes.
+
+`nodes.os.mem.used_percent`::
+(integer)
+Percentage of physical memory in use across all selected nodes.
+
+`nodes.process.cpu.percent`::
+(integer)
+Percentage of CPU used across all selected nodes. Returns `-1` if
+not supported.
+
+`nodes.process.open_file_descriptors.min`::
+(integer)
+Minimum number of concurrently open file descriptors across all selected nodes.
+Returns `-1` if not supported.
+
+`nodes.process.open_file_descriptors.max`::
+(integer)
+Maximum number of concurrently open file descriptors allowed across all selected
+nodes. Returns `-1` if not supported.
+
+`nodes.process.open_file_descriptors.avg`::
+(integer)
+Average number of concurrently open file descriptors. Returns `-1` if not
+supported.
+
+`nodes.jvm.max_uptime`::
+(<<time-units,time unit>>)
+Uptime duration since JVM last started.
+
+`nodes.jvm.max_uptime_in_millis`::
+(integer)
+Uptime duration, in milliseconds, since JVM last started.
+
+`nodes.jvm.versions`::
+(array of objects)
+Contains details about the JVM versions used by selected
+nodes.
+
+`nodes.jvm.versions.version`:::
+(string)
+Version of JVM used by one or more selected nodes.
+
+`nodes.jvm.versions.vm_name`:::
+(string)
+Name of the JVM.
+
+`nodes.jvm.versions.vm_version`:::
+(string)
+Full version number of JVM.
++
+The full version number includes a plus sign (`+`) followed by the build number.
 
+`nodes.jvm.versions.vm_vendor`:::
+(string)
+Vendor of the JVM.
+
+`nodes.jvm.versions.bundled_jdk`:::
+(boolean)
+If `true`, the JVM includes a bundled Java Development Kit (JDK).
+
+`nodes.jvm.versions.using_bundled_jdk`:::
+(boolean)
+If `true`, a bundled JDK is in use by JVM.
+
+`nodes.jvm.versions.count`:::
+(integer)
+Total number of selected nodes using JVM.
+
+`nodes.jvm.mem.heap_used`::
+(<<byte-units, byte units>>)
+Memory currently in use by the heap across all selected nodes.
+
+`nodes.jvm.mem.heap_used_in_bytes`::
+(integer)
+Memory, in bytes, currently in use by the heap across all selected nodes.
+
+`nodes.jvm.mem.heap_max`::
+(<<byte-units, byte units>>)
+Maximum amount of memory, in bytes, available for use by the heap across all
+selected nodes.
+
+`nodes.jvm.mem.heap_max_in_bytes`::
+(integer)
+Maximum amount of memory, in bytes, available for use by the heap across all
+selected nodes.
+
+`nodes.jvm.threads`::
+(integer)
+Number of active threads in use by JVM across all selected nodes.
+
+`nodes.fs.total`::
+(<<byte-units, byte units>>)
+Total size of all file stores across all selected nodes.
+
+`nodes.fs.total_in_bytes`::
+(integer)
+Total size, in bytes, of all file stores across all seleced nodes.
+
+`nodes.fs.free`::
+(<<byte-units, byte units>>)
+Amount of unallocated disk space in file stores across all selected nodes.
+
+`nodes.fs.free_in_bytes`::
+(integer)
+Total number of unallocated bytes in file stores across all selected nodes.
+
+`nodes.fs.available`::
+(<<byte-units, byte units>>)
+Total amount of disk space available to JVM in file
+stores across all selected nodes.
++
+Depending on OS or process-level restrictions, this amount may be less than
+`nodes.fs.free`. This is the actual amount of free disk space the selected {es}
+nodes can use.
+
+`nodes.fs.available_in_bytes`::
+(integer)
+Total number of bytes available to JVM in file stores
+across all selected nodes.
++
+Depending on OS or process-level restrictions, this number may be less than
+`nodes.fs.free_in_byes`. This is the actual amount of free disk space the
+selected {es} nodes can use.
+
+`nodes.plugins`::
+(array of objects)
+Contains details about installed plugins and modules across all selected nodes.
++
+If no plugins or modules are installed, this array is empty.
+
+`nodes.plugins.name`:::
+(string)
+Name of the {es} plugin.
+
+`nodes.plugins.version`:::
+(string)
+{es} version for which the plugin was built.
+
+`nodes.plugins.elasticsearch_version`:::
+(string)
+{es} version for which the plugin was built.
+
+`node.plugins.java_version`:::
+(string)
+Java version for which the plugin was built.
+
+`nodes.plugins.description`:::
+(string)
+Short description of the plugin.
+
+`nodes.plugins.classname`:::
+(string)
+Class name used as the plugin's entry point.
+
+`nodes.plugins.extended_plugins`:::
+(array of strings)
+An array of other plugins extended by this plugin through the Java Service
+Provider Interface (SPI).
++
+If this plugin extends no other plugins, this array is empty.
+
+`nodes.plugins.has_native_controller`:::
+(boolean)
+If `true`, the plugin has a native controller process.
+
+`nodes.network_types.transport_types.<transport_type>`::
+(integer)
+Number of selected nodes using the transport type.
+
+`nodes.network_types.http_types.<http_type>`::
+(integer)
+Number of selected nodes using the HTTP type.
+
+`nodes.discovery_types.<discovery_type>`::
+(integer)
+Number of selected nodes using the <<modules-discovery-hosts-providers,discovery
+type>> to find other nodes.
+
+`nodes.packaging_types`::
+(array of objects)
+Contains details about {es} distributions installed on selected nodes.
+
+`nodes.packaging_types.flavor`:::
+(string)
+Type of {es} distribution, such as `default` or `OSS`, used by one or more
+selected nodes.
+
+`nodes.packaging_types.type`:::
+(string)
+File type, such as `tar` or `zip`, used for the distribution package.
+
+`nodes.packaging_types.count`:::
+(integer)
+Number of selected nodes using the distribution flavor and file type.
+====
 
 [[cluster-stats-api-example]]
 ==== {api-examples-title}
diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc
index 190734198c77d..ccb3752854e8b 100644
--- a/docs/reference/eql/search.asciidoc
+++ b/docs/reference/eql/search.asciidoc
@@ -29,6 +29,15 @@ PUT sec_logs/_bulk?refresh
 ----
 // TESTSETUP
 
+[TIP]
+====
+You also can set up {beats-ref}/getting-started.html[{beats}], such as
+{auditbeat-ref}/auditbeat-getting-started.html[{auditbeat}] or
+{winlogbeat-ref}/winlogbeat-getting-started.html[{winlogbeat}], to automatically
+send and index your event data in {es}. See
+{beats-ref}/getting-started.html[Getting started with {beats}].
+====
+
 You can now use the EQL search API to search this index using an EQL query.
 
 The following request searches the `sec_logs` index using the EQL query
diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc
index 40ff0828afe2d..fac44f8bcdc53 100644
--- a/docs/reference/glossary.asciidoc
+++ b/docs/reference/glossary.asciidoc
@@ -24,28 +24,55 @@ Also see <<glossary-text,text>> and <<glossary-term,term>>.
 // end::analysis-def[]
 --
 
+[[glossary-api-key]] API key ::
+// tag::api-key-def[]
+A unique identifier that you can use for authentication when submitting {es} requests. 
+When TLS is enabled, all requests must be authenticated using either basic authentication 
+(user name and password) or an API key.
+// end::api-key-def[]
+
+
+[[glossary-auto-follow-pattern]] auto-follow pattern ::
+// tag::auto-follow-pattern-def[]
+An <<glossary-index-pattern,index pattern>> that automatically configures new indices as 
+<<glossary-follower-index,follower indices>> for <<glossary-ccr,{ccr}>>.
+For more information, see {ref}/ccr-auto-follow.html#_managing_auto_follow_patterns[Managing auto follow patterns].
+// end::auto-follow-pattern-def[]
+
 [[glossary-cluster]] cluster ::
 // tag::cluster-def[]
-A cluster consists of one or more <<glossary-node,nodes>> which share the
-same cluster name. Each cluster has a single master node which is
-chosen automatically by the cluster and which can be replaced if the
-current master node fails.
+One or more <<glossary-node,nodes>> that share the
+same cluster name. Each cluster has a single master node, which is
+chosen automatically by the cluster and can be replaced if it fails.
 // end::cluster-def[]
 
+[[glossary-cold-phase]] cold phase ::
+// tag::cold-phase-def[]
+The third possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the cold phase, an index is no longer updated and seldom queried. 
+The information still needs to be searchable, but it’s okay if those queries are slower.
+// end::cold-phase-def[]
+
 [[glossary-ccr]] {ccr} (CCR)::
 // tag::ccr-def[]
-The {ccr} feature enables you to replicate indices in remote clusters to your
+A feature that enables you to replicate indices in remote clusters to your
 local cluster. For more information, see 
 {ref}/xpack-ccr.html[{ccr-cap}].  
 // end::ccr-def[]
   
 [[glossary-ccs]] {ccs} (CCS)::
 // tag::ccs-def[]
-The {ccs} feature enables any node to act as a federated client across
+A feature that enables any node to act as a federated client across
 multiple clusters.
 See {ref}/modules-cross-cluster-search.html[Search across clusters].
 // end::ccs-def[]
 
+[[glossary-delete-phase]] delete phase ::
+// tag::delete-phase-def[]
+The last possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the delete phase, an index is no longer needed and can safely be deleted.
+// end::delete-phase-def[]
+
 [[glossary-document]] document ::
 +
 --
@@ -93,10 +120,22 @@ hence it is called a filter. Filters are simple checks for set inclusion or excl
 In most cases, the goal of filtering is to reduce the number of documents that have to be examined.
 // end::filter-def[]
 
+[[glossary-flush]] flush ::
+// tag::flush-def[]
+Peform a Lucene commit to write index updates in the transaction log (translog) to disk. 
+Because a Lucene commit is a relatively expensive operation, 
+{es} records index and delete operations in the translog and 
+automatically flushes changes to disk in batches.
+To recover from a crash, operations that have been acknowledged but not yet committed 
+can be replayed from the translog.
+Before upgrading, you can explicitly call the {ref}/indices-flush.html[Flush] API
+to ensure that all changes are committed to disk.
+// end::flush-def[]
+
 [[glossary-follower-index]] follower index ::  
 // tag::follower-index-def[]
-Follower indices are the target indices for <<glossary-ccr,{ccr}>>. They exist
-in your local cluster and replicate <<glossary-leader-index,leader indices>>.
+The target index for <<glossary-ccr,{ccr}>>. A follower index exists
+in a local cluster and replicates a <<glossary-leader-index,leader index>>.
 // end::follower-index-def[]
 
 [[glossary-force-merge]] force merge ::  
@@ -107,7 +146,7 @@ and free up the space used by deleted documents.
 // end::force-merge-def-short[]
 You should not force merge indices that are actively being written to.
 Merging is normally performed automatically, but you can use force merge after 
-<<glossary-rollover, rollover>> to reduce the shards in the old index to a single segment. 
+<<glossary-rollover,rollover>> to reduce the shards in the old index to a single segment. 
 See the {ref}/indices-forcemerge.html[force merge API].
 // end::force-merge-def[]
 
@@ -123,6 +162,19 @@ before you are ready to archive or delete them.
 See the {ref}/freeze-index-api.html[freeze API].
 // end::freeze-def[]
 
+[[glossary-frozen-index]] frozen index ::  
+// tag::frozen-index-def[]
+An index reduced to a low overhead state that still enables occasional searches.
+Frozen indices use a memory-efficient shard implementation and throttle searches to conserve resources.
+Searching a frozen index is lower overhead than re-opening a closed index to enable searching.
+// end::frozen-index-def[]
+
+[[glossary-hot-phase]] hot phase ::
+// tag::hot-phase-def[]
+The first possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the hot phase, an index is actively updated and queried. 
+// end::hot-phase-def[]
+
 [[glossary-id]] id ::
 // tag::id-def[]
 The ID of a <<glossary-document,document>> identifies a document. The
@@ -134,11 +186,12 @@ then it will be auto-generated. (also see <<glossary-routing,routing>>)
 +
 --
 // tag::index-def[]
-An index is like a _table_ in a relational database. It has a
-<<glossary-mapping,mapping>> which contains a <<glossary-type,type>>,
-which contains the <<glossary-field,field>> in the index.
+// tag::index-def-short[]
+An optimized collection of JSON documents. Each document is a collection of fields, 
+the key-value pairs that contain your data. 
+// end::index-def-short[]
 
-An index is a logical namespace which maps to one or more
+An index is a logical namespace that maps to one or more
 <<glossary-primary-shard,primary shards>> and can have zero or more
 <<glossary-replica-shard,replica shards>>.
 // end::index-def[]
@@ -149,17 +202,38 @@ An index is a logical namespace which maps to one or more
 --
 // tag::index-alias-def[]
 // tag::index-alias-desc[]
-An index alias is a secondary name
-used to refer to one or more existing indices.
+An index alias is a logical name used to reference one or more indices.
 
-Most {es} APIs accept an index alias
-in place of an index name.
+Most {es} APIs accept an index alias in place of an index name.
 // end::index-alias-desc[]
 
 See {ref}/indices-add-alias.html[Add index alias].
 // end::index-alias-def[]
 --
 
+[[glossary-index-lifecycle]] index lifecycle ::
+// tag::index-lifecycle-def[]
+The four phases an index can transition through: 
+<<glossary-hot-phase,hot>>, <<glossary-warm-phase,warm>>,
+<<glossary-cold-phase,cold>>, and <<glossary-delete-phase,delete>>.
+For more information, see {ref}/ilm-policy-definition.html[Index lifecycle].
+// end::index-lifecycle-def[]
+
+[[glossary-index-lifecycle-policy]] index lifecycle policy ::
+// tag::index-lifecycle-policy-def[]
+Specifies how an index moves between phases in the index lifecycle and 
+what actions to perform during each phase. 
+// end::index-lifecycle-policy-def[]
+
+[[glossary-index-pattern]] index pattern ::
+// tag::index-pattern-def[]
+A string that can contain the `*` wildcard to match multiple index names.
+In most cases, the index parameter in an {es} request can be the name of a specific index, 
+a list of index names, or an index pattern.
+For example, if you have the indices `datastream-000001`, `datastream-000002`, and `datastream-000003`,
+to search across all three you could use the `datastream-*` index pattern.  
+// end::index-pattern-def[]
+
 [[glossary-index-template]] index template ::
 +
 --
@@ -167,18 +241,24 @@ See {ref}/indices-add-alias.html[Add index alias].
 // tag::index-template-def-short[]
 Defines settings and mappings to apply to new indexes that match a simple naming pattern, such as _logs-*_. 
 // end::index-template-def-short[]
+
 An index template can also attach a lifecycle policy to the new index.
-Index templates are used to automatically configure indices created during <<glossary-rollover, rollover>>. 
+Index templates are used to automatically configure indices created during <<glossary-rollover,rollover>>. 
 // end::index-template-def[]
 --
 
 [[glossary-leader-index]] leader index ::  
 // tag::leader-index-def[]
-Leader indices are the source indices for <<glossary-ccr,{ccr}>>. They exist
-on remote clusters and are replicated to 
+The source index for <<glossary-ccr,{ccr}>>. A leader index exists
+on a remote cluster and is replicated to 
 <<glossary-follower-index,follower indices>>.
 // end::leader-index-def[]
 
+[[glossary-local-cluster]] local cluster ::
+// tag::local-cluster-def[]
+The cluster that pulls data from a <<glossary-remote-cluster,remote cluster>> in {ccs} or {ccr}.
+// end::local-cluster-def[]
+
 [[glossary-mapping]] mapping ::
 +
 --
@@ -194,18 +274,12 @@ automatically when a document is indexed.
 --
 
 [[glossary-node]] node ::
-+
---
 // tag::node-def[]
-A node is a running instance of Elasticsearch which belongs to a
+A running instance of {es} that belongs to a
 <<glossary-cluster,cluster>>. Multiple nodes can be started on a single
 server for testing purposes, but usually you should have one node per
 server.
-
-At startup, a node will use unicast to discover an existing cluster with
-the same cluster name and will try to join that cluster.
 // end::node-def[]
---
 
 [[glossary-primary-shard]] primary shard ::
 +
@@ -267,9 +341,18 @@ during the following processes:
 [[glossary-reindex]] reindex ::
 
 // tag::reindex-def[]
-To cycle through some or all documents in one or more indices, re-writing them into the same or new index in a local or remote cluster. This is most commonly done to update mappings, or to upgrade Elasticsearch between two incompatible index versions.
+To cycle through some or all documents in one or more indices, re-writing them into the same 
+or new index in a local or remote cluster. This is most commonly done to update mappings, or to upgrade {es} between two incompatible index versions.
 // end::reindex-def[]
 
+[[glossary-remote-cluster]] remote cluster ::
+
+// tag::remote-cluster-def[]
+A separate cluster, often in a different data center or locale, that contains indices that 
+can be replicated or searched by the <<glossary-local-cluster,local cluster>>. 
+The connection to a remote cluster is unidirectional.
+// end::remote-cluster-def[]
+
 [[glossary-replica-shard]] replica shard ::
 +
 --
@@ -294,14 +377,35 @@ shard will never be started on the same node as its primary shard.
 --
 // tag::rollover-def[]
 // tag::rollover-def-short[]
-Redirect an alias to begin writing to a new index when the existing index reaches a certain age, number of docs, or size. 
+Redirect an index alias to begin writing to a new index when the existing index reaches 
+a certain size, number of docs, or age. 
 // end::rollover-def-short[]
-The new index is automatically configured according to any matching <<glossary-index-template, index templates>>. 
+
+The new index is automatically configured according to any matching <<glossary-index-template,index templates>>. 
 For example, if you're indexing log data, you might use rollover to create daily or weekly indices. 
 See the {ref}/indices-rollover-index.html[rollover index API].
 // end::rollover-def[]
 --
 
+[[glossary-rollup]] rollup ::
+// tag::rollup-def[]
+Summarize high-granularity data into a more compressed format to 
+maintain access to historical data in a cost-effective way.
+// end::rollup-def[]
+
+[[glossary-rollup-index]] rollup index ::
+// tag::rollup-index-def[]
+A special type of index for storing historical data at reduced granularity. 
+Documents are summarized and indexed into a rollup index by a <<glossary-rollup-job,rollup job>>.
+// end::rollup-index-def[]
+
+[[glossary-rollup-job]] rollup job ::
+// tag::rollup-job-def[]
+A background task that runs continuously to summarize documents in an index and 
+index the summaries into a separate rollup index. 
+The job configuration controls what information is rolled up and how often. 
+// end::rollup-job-def[]
+
 [[glossary-routing]] routing ::
 +
 --
@@ -340,14 +444,36 @@ nodes.
 --
 
 [[glossary-shrink]] shrink ::
++
+--
 // tag::shrink-def[]
 // tag::shrink-def-short[]
 Reduce the number of primary shards in an index.
 // end::shrink-def-short[]
+
 You can shrink an index to reduce its overhead when the request volume drops.
 For example, you might opt to shrink an index once it is no longer the write index. 
 See the {ref}/indices-shrink-index.html[shrink index API].
 // end::shrink-def[]
+--
+
+[[glossary-snapshot]] snapshot ::
+// tag::snapshot-def[]
+A backup taken from a running {es} cluster. 
+You can take snapshots of individual indices or of the entire cluster. 
+// end::snapshot-def[]
+
+[[glossary-snapshot-lifecycle-policy]] snapshot lifecycle policy ::
+// tag::snapshot-lifecycle-policy-def[]
+Specifies how frequently to perform automatic backups of a cluster and 
+how long to retain the resulting snapshots. 
+// end::snapshot-lifecycle-policy-def[]
+
+[[glossary-snapshot-repository]] snapshot repository ::
+// tag::snapshot-repository-def[]
+Specifies where snapshots are to be stored. 
+Snapshots can be written to a shared filesystem or to a remote repository.
+// end::snapshot-repository-def[]
 
 [[glossary-source_field]] source field ::
 // tag::source-field-def[]
@@ -399,3 +525,9 @@ A type used to represent the _type_ of document, e.g. an `email`, a `user`, or a
 Types are deprecated and are in the process of being removed.
 See {ref}/removal-of-types.html[Removal of mapping types].
 // end::type-def[]
+
+[[glossary-warm-phase]] warm phase ::
+// tag::warm-phase-def[]
+The second possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the warm phase, an index is generally optimized for search and no longer updated.
+// end::warm-phase-def[]
diff --git a/docs/reference/ilm/policy-definitions.asciidoc b/docs/reference/ilm/policy-definitions.asciidoc
index d66a53ecea55a..1c917eade49d2 100644
--- a/docs/reference/ilm/policy-definitions.asciidoc
+++ b/docs/reference/ilm/policy-definitions.asciidoc
@@ -113,7 +113,7 @@ policy definition.
   - <<ilm-allocate-action,Allocate>>
   - <<ilm-freeze-action,Freeze>>
 * Delete
-  - <<ilm-delete-action,Wait For Snapshot>>
+  - <<ilm-wait-for-snapshot-action,Wait For Snapshot>>
   - <<ilm-delete-action,Delete>>
 
 [[ilm-allocate-action]]
diff --git a/docs/reference/images/analysis/token-graph-basic.svg b/docs/reference/images/analysis/token-graph-basic.svg
new file mode 100644
index 0000000000000..99c2b0cb24f5c
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-basic.svg
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="602px" height="54px" viewBox="0 0 602 54" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="token-graph-basic" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
+                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
+                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="73.96063 10.47244 152.35432 10.47244 152.35432 41.842511 73.96063 41.842511"></polygon>
+                <path d="M84.39813,33.07748 L84.39813,19.718097 L89.445005,19.718097 C90.3304217,19.718097 91.007505,19.7597637 91.476255,19.843097 C92.1220883,19.9576803 92.6689633,20.171222 93.11688,20.483722 C93.5647967,20.7858053 93.9241717,21.2128887 94.195005,21.764972 C94.4658383,22.3170553 94.601255,22.9264303 94.601255,23.593097 C94.601255,24.7285137 94.2366717,25.6920553 93.507505,26.483722 C92.788755,27.264972 91.4814633,27.655597 89.58563,27.655597 L86.163755,27.655597 L86.163755,33.07748 L84.39813,33.07748 L84.39813,33.07748 Z M86.163755,26.0774724 L89.61688,26.0774724 C90.7627133,26.0774724 91.5752133,25.8639307 92.05438,25.4368474 C92.5335467,25.0097641 92.77313,24.4108057 92.77313,23.6399724 C92.77313,23.0774724 92.632505,22.5983057 92.351255,22.2024724 C92.070005,21.8066391 91.695005,21.5462224 91.226255,21.4212224 C90.9345883,21.3378891 90.3877133,21.2962224 89.58563,21.2962224 L86.163755,21.2962224 L86.163755,26.0774724 L86.163755,26.0774724 Z M96.029203,28.23373 C96.029203,26.4420583 96.5239947,25.1191391 97.513578,24.2649724 C98.3469113,23.5462224 99.3625363,23.1868474 100.560453,23.1868474 C101.893786,23.1868474 102.982328,23.6243474 103.826078,24.4993474 C104.669828,25.3639307 105.091703,26.5670583 105.091703,28.10873 C105.091703,29.3483133 104.904203,30.32748 104.529203,31.04623 C104.154203,31.7545633 103.607328,32.3066467 102.888578,32.70248 C102.180245,33.0983133 101.404203,33.29623 100.560453,33.29623 C99.2062863,33.29623 98.1125363,32.8639383 97.279203,31.999355 C96.4458697,31.124355 96.029203,29.8691467 96.029203,28.23373 Z M97.716703,28.23373 C97.716703,29.4733133 97.982328,30.405605 98.513578,31.030605 C99.0552447,31.6451883 99.7375363,31.95248 100.560453,31.95248 C101.372953,31.95248 102.050036,31.6451883 102.591703,31.030605 C103.13337,30.405605 103.404203,29.4576883 103.404203,28.186855 C103.404203,26.9889333 103.13337,26.0826807 102.591703,25.4680974 C102.050036,24.8535141 101.372953,24.5462224 100.560453,24.5462224 C99.7375363,24.5462224 99.0552447,24.8535141 98.513578,25.4680974 C97.982328,26.0722641 97.716703,26.9941416 97.716703,28.23373 Z M108.795545,33.07748 L105.826795,23.405597 L107.52992,23.405597 L109.06117,28.9837296 L109.639295,31.0618546 C109.670545,30.9576879 109.837212,30.2910213 110.139295,29.0618546 L111.68617,23.405597 L113.37367,23.405597 L114.81117,29.0149796 L115.295545,30.8587296 L115.858045,28.9993546 L117.514295,23.405597 L119.108045,23.405597 L116.076795,33.07748 L114.37367,33.07748 L112.84242,27.2805974 L112.46742,25.6399724 L110.514295,33.07748 L108.795545,33.07748 Z M127.112241,29.968105 L128.799741,30.17123 C128.528908,31.1608133 128.034116,31.9316467 127.315366,32.48373 C126.596616,33.0253967 125.674741,33.29623 124.549741,33.29623 C123.133074,33.29623 122.008074,32.8639383 121.174741,31.999355 C120.351824,31.124355 119.940366,29.9003967 119.940366,28.32748 C119.940366,26.6920583 120.357033,25.4264307 121.190366,24.5305974 C122.034116,23.6347641 123.122658,23.1868474 124.455991,23.1868474 C125.747658,23.1868474 126.799741,23.6295557 127.612241,24.5149724 C128.435158,25.3899724 128.846616,26.6243499 128.846616,28.218105 C128.846616,28.3222717 128.846616,28.468105 128.846616,28.655605 L121.627866,28.655605 C121.690366,29.718105 121.992449,30.5358133 122.534116,31.10873 C123.075783,31.67123 123.747658,31.95248 124.549741,31.95248 C125.153908,31.95248 125.669533,31.79623 126.096616,31.48373 C126.523699,31.1608133 126.862241,30.655605 127.112241,29.968105 Z M121.721616,27.3118474 L127.127866,27.3118474 C127.054949,26.4993474 126.846616,25.8899724 126.502866,25.4837224 C125.982033,24.8483057 125.304949,24.5305974 124.471616,24.5305974 C123.721616,24.5305974 123.086199,24.7858057 122.565366,25.2962224 C122.054949,25.7962224 121.773699,26.4680974 121.721616,27.3118474 Z M130.847333,33.07748 L130.847333,23.405597 L132.316083,23.405597 L132.316083,24.874347 C132.691083,24.186847 133.034833,23.733722 133.347333,23.514972 C133.67025,23.296222 134.024416,23.186847 134.409833,23.186847 C134.961916,23.186847 135.524416,23.3639303 136.097333,23.718097 L135.534833,25.233722 C135.128583,24.9941387 134.727541,24.874347 134.331708,24.874347 C133.967125,24.874347 133.644208,24.983722 133.362958,25.202472 C133.081708,25.421222 132.878583,25.718097 132.753583,26.093097 C132.566083,26.6764303 132.472333,27.3170579 132.472333,28.0149796 L132.472333,33.07748 L130.847333,33.07748 L130.847333,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="265.96063 10.47244 332.38583 10.47244 332.38583 41.842511 265.96063 41.842511"></polygon>
+                <path d="M280.82782,28.78061 L282.48407,28.639985 C282.567403,29.3066517 282.749695,29.8535267 283.030945,30.28061 C283.322612,30.7076933 283.770528,31.0566517 284.374695,31.327485 C284.989278,31.5879017 285.676778,31.71811 286.437195,31.71811 C287.103862,31.71811 287.697612,31.6191517 288.218445,31.421235 C288.739278,31.2233183 289.124695,30.952485 289.374695,30.608735 C289.624695,30.2545683 289.749695,29.8691517 289.749695,29.452485 C289.749695,29.0358183 289.624695,28.671235 289.374695,28.358735 C289.135112,28.0462299 288.739278,27.7806024 288.187195,27.5618524 C287.822612,27.4264357 287.025737,27.2128941 285.79657,26.9212274 C284.57782,26.6191441 283.723653,26.3378941 283.23407,26.0774774 C282.588237,25.7441441 282.10907,25.3326857 281.79657,24.8431024 C281.48407,24.3431024 281.32782,23.7858107 281.32782,23.1712274 C281.32782,22.5045607 281.520528,21.8795607 281.905945,21.2962274 C282.291362,20.7024774 282.848653,20.2545607 283.57782,19.9524774 C284.317403,19.6503941 285.135112,19.4993524 286.030945,19.4993524 C287.020528,19.4993524 287.89032,19.6608107 288.64032,19.9837274 C289.400737,20.2962274 289.98407,20.7649774 290.39032,21.3899774 C290.79657,22.0045607 291.01532,22.7024774 291.04657,23.4837274 L289.35907,23.6087274 C289.26532,22.7649774 288.95282,22.1295607 288.42157,21.7024774 C287.900737,21.2649774 287.129903,21.0462274 286.10907,21.0462274 C285.036153,21.0462274 284.254903,21.2441441 283.76532,21.6399774 C283.275737,22.0358107 283.030945,22.5097691 283.030945,23.0618524 C283.030945,23.5410191 283.208028,23.9316441 283.562195,24.2337274 C283.895528,24.5462274 284.780945,24.8691441 286.218445,25.2024774 C287.655945,25.5253941 288.64032,25.8066441 289.17157,26.0462274 C289.95282,26.4003941 290.525737,26.8535191 290.89032,27.4056024 C291.26532,27.9576908 291.45282,28.59311 291.45282,29.31186 C291.45282,30.0201933 291.249695,30.6920683 290.843445,31.327485 C290.437195,31.952485 289.853862,32.4420683 289.093445,32.796235 C288.333028,33.139985 287.473653,33.31186 286.51532,33.31186 C285.306987,33.31186 284.291362,33.1347767 283.468445,32.78061 C282.655945,32.4264433 282.01532,31.8951933 281.54657,31.18686 C281.088237,30.4785267 280.848653,29.6764433 280.82782,28.78061 Z M293.662018,33.077485 L293.662018,19.718102 L295.302643,19.718102 L295.302643,24.514977 C296.06306,23.6295603 297.026601,23.186852 298.193268,23.186852 C298.912018,23.186852 299.53181,23.327477 300.052643,23.608727 C300.583893,23.889977 300.964101,24.280602 301.193268,24.780602 C301.422435,25.280602 301.537018,26.0045603 301.537018,26.952477 L301.537018,33.077485 L299.896393,33.077485 L299.896393,26.952477 C299.896393,26.1295603 299.71931,25.530602 299.365143,25.155602 C299.010976,24.780602 298.505768,24.593102 297.849518,24.593102 C297.370351,24.593102 296.917226,24.7233103 296.490143,24.983727 C296.06306,25.233727 295.755768,25.5722687 295.568268,25.999352 C295.391185,26.4264353 295.302643,27.0201853 295.302643,27.780602 L295.302643,33.077485 L293.662018,33.077485 L293.662018,33.077485 Z M303.428375,28.233735 C303.428375,26.4420633 303.923167,25.1191441 304.91275,24.2649774 C305.746083,23.5462274 306.761708,23.1868524 307.959625,23.1868524 C309.292958,23.1868524 310.3815,23.6243524 311.22525,24.4993524 C312.069,25.3639357 312.490875,26.5670633 312.490875,28.108735 C312.490875,29.3483183 312.303375,30.327485 311.928375,31.046235 C311.553375,31.7545683 311.0065,32.3066517 310.28775,32.702485 C309.579417,33.0983183 308.803375,33.296235 307.959625,33.296235 C306.605458,33.296235 305.511708,32.8639433 304.678375,31.99936 C303.845042,31.12436 303.428375,29.8691517 303.428375,28.233735 Z M305.115875,28.233735 C305.115875,29.4733183 305.3815,30.40561 305.91275,31.03061 C306.454417,31.6451933 307.136708,31.952485 307.959625,31.952485 C308.772125,31.952485 309.449208,31.6451933 309.990875,31.03061 C310.532542,30.40561 310.803375,29.4576933 310.803375,28.18686 C310.803375,26.9889383 310.532542,26.0826857 309.990875,25.4681024 C309.449208,24.8535191 308.772125,24.5462274 307.959625,24.5462274 C307.136708,24.5462274 306.454417,24.8535191 305.91275,25.4681024 C305.3815,26.0722691 305.115875,26.9941466 305.115875,28.233735 Z M317.991577,31.608735 L318.225952,33.06186 C317.767619,33.15561 317.35616,33.202485 316.991577,33.202485 C316.397827,33.202485 315.934285,33.108735 315.600952,32.921235 C315.278035,32.733735 315.048869,32.4889433 314.913452,32.18686 C314.778035,31.87436 314.710327,31.2285267 314.710327,30.24936 L314.710327,24.6712274 L313.507202,24.6712274 L313.507202,23.4056024 L314.710327,23.4056024 L314.710327,21.0149774 L316.335327,20.0306024 L316.335327,23.4056024 L317.991577,23.4056024 L317.991577,24.6712274 L316.335327,24.6712274 L316.335327,30.34311 C316.335327,30.8014433 316.361369,31.0983183 316.413452,31.233735 C316.475952,31.3691517 316.569702,31.4785267 316.694702,31.56186 C316.830119,31.6347767 317.022827,31.671235 317.272827,31.671235 C317.44991,31.671235 317.689494,31.6504017 317.991577,31.608735 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="447.53366 10.47244 518.43127 10.47244 518.43127 41.842511 447.53366 41.842511"></polygon>
+                <path d="M471.62164,31.49936 L471.62164,33.077485 L462.793545,33.077485 C462.783129,32.6816517 462.845629,32.3014433 462.981045,31.93686 C463.210212,31.3326933 463.569587,30.7389433 464.05917,30.15561 C464.55917,29.5722767 465.27792,28.9004017 466.21542,28.139985 C467.663337,26.9524799 468.642504,26.0097691 469.15292,25.3118524 C469.663337,24.6139357 469.918545,23.9576857 469.918545,23.3431024 C469.918545,22.6868524 469.68417,22.1399774 469.21542,21.7024774 C468.757087,21.2545607 468.15292,21.0306024 467.40292,21.0306024 C466.611254,21.0306024 465.975837,21.2701857 465.49667,21.7493524 C465.02792,22.2181024 464.793545,22.8691441 464.793545,23.7024774 L463.106045,23.5306024 C463.220629,22.2701857 463.65292,21.3118524 464.40292,20.6556024 C465.163337,19.9993524 466.173754,19.6712274 467.43417,19.6712274 C468.71542,19.6712274 469.731045,20.0253941 470.481045,20.7337274 C471.231025,21.4420607 471.606015,22.3222691 471.606015,23.3743524 C471.606015,23.9056024 471.49664,24.4316441 471.27789,24.9524774 C471.05916,25.4733107 470.694587,26.0201857 470.18417,26.5931024 C469.68417,27.1556024 468.84042,27.9368549 467.65292,28.93686 C466.673754,29.7597767 466.043545,30.3222767 465.762295,30.62436 C465.481045,30.9160267 465.251879,31.2076933 465.074795,31.49936 L471.62164,31.49936 Z M473.387997,26.4837274 C473.387997,24.9108107 473.549456,23.6451857 473.872372,22.6868524 C474.195289,21.7181024 474.674456,20.9733107 475.309872,20.4524774 C475.955706,19.9316441 476.762997,19.6712274 477.731747,19.6712274 C478.450497,19.6712274 479.080706,19.8170607 479.622372,20.1087274 C480.164039,20.3899774 480.606747,20.8066441 480.950497,21.3587274 C481.304664,21.9003941 481.580706,22.5618524 481.778622,23.3431024 C481.986956,24.1139357 482.091122,25.1608107 482.091122,26.4837274 C482.091122,28.0566491 481.929664,29.327485 481.606747,30.296235 C481.283831,31.2545683 480.799456,31.99936 480.153622,32.53061 C479.518206,33.0514433 478.710914,33.31186 477.731747,33.31186 C476.450497,33.31186 475.440081,32.8483183 474.700497,31.921235 C473.825497,30.8066517 473.387997,28.9941491 473.387997,26.4837274 Z M475.059872,26.4837274 C475.059872,28.6816491 475.315081,30.1451933 475.825497,30.87436 C476.346331,31.59311 476.981747,31.952485 477.731747,31.952485 C478.492164,31.952485 479.127581,31.5879017 479.637997,30.858735 C480.148414,30.1295683 480.403622,28.6712325 480.403622,26.4837274 C480.403622,24.2858107 480.148414,22.8274774 479.637997,22.1087274 C479.127581,21.3899774 478.486956,21.0306024 477.716122,21.0306024 C476.966122,21.0306024 476.367164,21.3483107 475.919247,21.9837274 C475.346331,22.7962274 475.059872,24.2962274 475.059872,26.4837274 Z M483.763729,26.4837274 C483.763729,24.9108107 483.925188,23.6451857 484.248104,22.6868524 C484.571021,21.7181024 485.050188,20.9733107 485.685604,20.4524774 C486.331438,19.9316441 487.138729,19.6712274 488.107479,19.6712274 C488.826229,19.6712274 489.456438,19.8170607 489.998104,20.1087274 C490.539771,20.3899774 490.982479,20.8066441 491.326229,21.3587274 C491.680396,21.9003941 491.956438,22.5618524 492.154354,23.3431024 C492.362688,24.1139357 492.466854,25.1608107 492.466854,26.4837274 C492.466854,28.0566491 492.305396,29.327485 491.982479,30.296235 C491.659563,31.2545683 491.175188,31.99936 490.529354,32.53061 C489.893938,33.0514433 489.086646,33.31186 488.107479,33.31186 C486.826229,33.31186 485.815813,32.8483183 485.076229,31.921235 C484.201229,30.8066517 483.763729,28.9941491 483.763729,26.4837274 Z M485.435604,26.4837274 C485.435604,28.6816491 485.690813,30.1451933 486.201229,30.87436 C486.722063,31.59311 487.357479,31.952485 488.107479,31.952485 C488.867896,31.952485 489.503313,31.5879017 490.013729,30.858735 C490.524146,30.1295683 490.779354,28.6712325 490.779354,26.4837274 C490.779354,24.2858107 490.524146,22.8274774 490.013729,22.1087274 C489.503313,21.3899774 488.862688,21.0306024 488.091854,21.0306024 C487.341854,21.0306024 486.742896,21.3483107 486.294979,21.9837274 C485.722063,22.7962274 485.435604,24.2962274 485.435604,26.4837274 Z M494.139461,26.4837274 C494.139461,24.9108107 494.30092,23.6451857 494.623836,22.6868524 C494.946753,21.7181024 495.42592,20.9733107 496.061336,20.4524774 C496.70717,19.9316441 497.514461,19.6712274 498.483211,19.6712274 C499.201961,19.6712274 499.83217,19.8170607 500.373836,20.1087274 C500.915503,20.3899774 501.358211,20.8066441 501.701961,21.3587274 C502.056128,21.9003941 502.33217,22.5618524 502.530086,23.3431024 C502.73842,24.1139357 502.842586,25.1608107 502.842586,26.4837274 C502.842586,28.0566491 502.681128,29.327485 502.358211,30.296235 C502.035295,31.2545683 501.55092,31.99936 500.905086,32.53061 C500.26967,33.0514433 499.462378,33.31186 498.483211,33.31186 C497.201961,33.31186 496.191545,32.8483183 495.451961,31.921235 C494.576961,30.8066517 494.139461,28.9941491 494.139461,26.4837274 Z M495.811336,26.4837274 C495.811336,28.6816491 496.066545,30.1451933 496.576961,30.87436 C497.097795,31.59311 497.733211,31.952485 498.483211,31.952485 C499.243628,31.952485 499.879045,31.5879017 500.389461,30.858735 C500.899878,30.1295683 501.155086,28.6712325 501.155086,26.4837274 C501.155086,24.2858107 500.899878,22.8274774 500.389461,22.1087274 C499.879045,21.3899774 499.23842,21.0306024 498.467586,21.0306024 C497.717586,21.0306024 497.118628,21.3483107 496.670711,21.9837274 C496.097795,22.7962274 495.811336,24.2962274 495.811336,26.4837274 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-wd.svg b/docs/reference/images/analysis/token-graph-wd.svg
new file mode 100644
index 0000000000000..cdbbfb8a0845c
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-wd.svg
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="602px" height="123px" viewBox="0 0 602 123" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Analysis-(1)" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
+                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
+                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="15.088913 39.14516 34.52198 39.14516 34.52198 60.971927 15.088913 60.971927"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="199.08891 39.14516 218.52197 39.14516 218.52197 60.971927 199.08891 60.971927"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="73.96063 10.47244 152.35432 10.47244 152.35432 41.842511 73.96063 41.842511"></polygon>
+                <path d="M84.39813,33.07748 L84.39813,19.718097 L89.445005,19.718097 C90.3304217,19.718097 91.007505,19.7597637 91.476255,19.843097 C92.1220883,19.9576803 92.6689633,20.171222 93.11688,20.483722 C93.5647967,20.7858053 93.9241717,21.2128887 94.195005,21.764972 C94.4658383,22.3170553 94.601255,22.9264303 94.601255,23.593097 C94.601255,24.7285137 94.2366717,25.6920553 93.507505,26.483722 C92.788755,27.264972 91.4814633,27.655597 89.58563,27.655597 L86.163755,27.655597 L86.163755,33.07748 L84.39813,33.07748 L84.39813,33.07748 Z M86.163755,26.0774724 L89.61688,26.0774724 C90.7627133,26.0774724 91.5752133,25.8639307 92.05438,25.4368474 C92.5335467,25.0097641 92.77313,24.4108057 92.77313,23.6399724 C92.77313,23.0774724 92.632505,22.5983057 92.351255,22.2024724 C92.070005,21.8066391 91.695005,21.5462224 91.226255,21.4212224 C90.9345883,21.3378891 90.3877133,21.2962224 89.58563,21.2962224 L86.163755,21.2962224 L86.163755,26.0774724 L86.163755,26.0774724 Z M96.029203,28.23373 C96.029203,26.4420583 96.5239947,25.1191391 97.513578,24.2649724 C98.3469113,23.5462224 99.3625363,23.1868474 100.560453,23.1868474 C101.893786,23.1868474 102.982328,23.6243474 103.826078,24.4993474 C104.669828,25.3639307 105.091703,26.5670583 105.091703,28.10873 C105.091703,29.3483133 104.904203,30.32748 104.529203,31.04623 C104.154203,31.7545633 103.607328,32.3066467 102.888578,32.70248 C102.180245,33.0983133 101.404203,33.29623 100.560453,33.29623 C99.2062863,33.29623 98.1125363,32.8639383 97.279203,31.999355 C96.4458697,31.124355 96.029203,29.8691467 96.029203,28.23373 Z M97.716703,28.23373 C97.716703,29.4733133 97.982328,30.405605 98.513578,31.030605 C99.0552447,31.6451883 99.7375363,31.95248 100.560453,31.95248 C101.372953,31.95248 102.050036,31.6451883 102.591703,31.030605 C103.13337,30.405605 103.404203,29.4576883 103.404203,28.186855 C103.404203,26.9889333 103.13337,26.0826807 102.591703,25.4680974 C102.050036,24.8535141 101.372953,24.5462224 100.560453,24.5462224 C99.7375363,24.5462224 99.0552447,24.8535141 98.513578,25.4680974 C97.982328,26.0722641 97.716703,26.9941416 97.716703,28.23373 Z M108.795545,33.07748 L105.826795,23.405597 L107.52992,23.405597 L109.06117,28.9837296 L109.639295,31.0618546 C109.670545,30.9576879 109.837212,30.2910213 110.139295,29.0618546 L111.68617,23.405597 L113.37367,23.405597 L114.81117,29.0149796 L115.295545,30.8587296 L115.858045,28.9993546 L117.514295,23.405597 L119.108045,23.405597 L116.076795,33.07748 L114.37367,33.07748 L112.84242,27.2805974 L112.46742,25.6399724 L110.514295,33.07748 L108.795545,33.07748 Z M127.112241,29.968105 L128.799741,30.17123 C128.528908,31.1608133 128.034116,31.9316467 127.315366,32.48373 C126.596616,33.0253967 125.674741,33.29623 124.549741,33.29623 C123.133074,33.29623 122.008074,32.8639383 121.174741,31.999355 C120.351824,31.124355 119.940366,29.9003967 119.940366,28.32748 C119.940366,26.6920583 120.357033,25.4264307 121.190366,24.5305974 C122.034116,23.6347641 123.122658,23.1868474 124.455991,23.1868474 C125.747658,23.1868474 126.799741,23.6295557 127.612241,24.5149724 C128.435158,25.3899724 128.846616,26.6243499 128.846616,28.218105 C128.846616,28.3222717 128.846616,28.468105 128.846616,28.655605 L121.627866,28.655605 C121.690366,29.718105 121.992449,30.5358133 122.534116,31.10873 C123.075783,31.67123 123.747658,31.95248 124.549741,31.95248 C125.153908,31.95248 125.669533,31.79623 126.096616,31.48373 C126.523699,31.1608133 126.862241,30.655605 127.112241,29.968105 Z M121.721616,27.3118474 L127.127866,27.3118474 C127.054949,26.4993474 126.846616,25.8899724 126.502866,25.4837224 C125.982033,24.8483057 125.304949,24.5305974 124.471616,24.5305974 C123.721616,24.5305974 123.086199,24.7858057 122.565366,25.2962224 C122.054949,25.7962224 121.773699,26.4680974 121.721616,27.3118474 Z M130.847333,33.07748 L130.847333,23.405597 L132.316083,23.405597 L132.316083,24.874347 C132.691083,24.186847 133.034833,23.733722 133.347333,23.514972 C133.67025,23.296222 134.024416,23.186847 134.409833,23.186847 C134.961916,23.186847 135.524416,23.3639303 136.097333,23.718097 L135.534833,25.233722 C135.128583,24.9941387 134.727541,24.874347 134.331708,24.874347 C133.967125,24.874347 133.644208,24.983722 133.362958,25.202472 C133.081708,25.421222 132.878583,25.718097 132.753583,26.093097 C132.566083,26.6764303 132.472333,27.3170579 132.472333,28.0149796 L132.472333,33.07748 L130.847333,33.07748 L130.847333,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="265.96063 10.47244 332.38583 10.47244 332.38583 41.842511 265.96063 41.842511"></polygon>
+                <path d="M280.82782,28.78061 L282.48407,28.639985 C282.567403,29.3066517 282.749695,29.8535267 283.030945,30.28061 C283.322612,30.7076933 283.770528,31.0566517 284.374695,31.327485 C284.989278,31.5879017 285.676778,31.71811 286.437195,31.71811 C287.103862,31.71811 287.697612,31.6191517 288.218445,31.421235 C288.739278,31.2233183 289.124695,30.952485 289.374695,30.608735 C289.624695,30.2545683 289.749695,29.8691517 289.749695,29.452485 C289.749695,29.0358183 289.624695,28.671235 289.374695,28.358735 C289.135112,28.0462299 288.739278,27.7806024 288.187195,27.5618524 C287.822612,27.4264357 287.025737,27.2128941 285.79657,26.9212274 C284.57782,26.6191441 283.723653,26.3378941 283.23407,26.0774774 C282.588237,25.7441441 282.10907,25.3326857 281.79657,24.8431024 C281.48407,24.3431024 281.32782,23.7858107 281.32782,23.1712274 C281.32782,22.5045607 281.520528,21.8795607 281.905945,21.2962274 C282.291362,20.7024774 282.848653,20.2545607 283.57782,19.9524774 C284.317403,19.6503941 285.135112,19.4993524 286.030945,19.4993524 C287.020528,19.4993524 287.89032,19.6608107 288.64032,19.9837274 C289.400737,20.2962274 289.98407,20.7649774 290.39032,21.3899774 C290.79657,22.0045607 291.01532,22.7024774 291.04657,23.4837274 L289.35907,23.6087274 C289.26532,22.7649774 288.95282,22.1295607 288.42157,21.7024774 C287.900737,21.2649774 287.129903,21.0462274 286.10907,21.0462274 C285.036153,21.0462274 284.254903,21.2441441 283.76532,21.6399774 C283.275737,22.0358107 283.030945,22.5097691 283.030945,23.0618524 C283.030945,23.5410191 283.208028,23.9316441 283.562195,24.2337274 C283.895528,24.5462274 284.780945,24.8691441 286.218445,25.2024774 C287.655945,25.5253941 288.64032,25.8066441 289.17157,26.0462274 C289.95282,26.4003941 290.525737,26.8535191 290.89032,27.4056024 C291.26532,27.9576908 291.45282,28.59311 291.45282,29.31186 C291.45282,30.0201933 291.249695,30.6920683 290.843445,31.327485 C290.437195,31.952485 289.853862,32.4420683 289.093445,32.796235 C288.333028,33.139985 287.473653,33.31186 286.51532,33.31186 C285.306987,33.31186 284.291362,33.1347767 283.468445,32.78061 C282.655945,32.4264433 282.01532,31.8951933 281.54657,31.18686 C281.088237,30.4785267 280.848653,29.6764433 280.82782,28.78061 Z M293.662018,33.077485 L293.662018,19.718102 L295.302643,19.718102 L295.302643,24.514977 C296.06306,23.6295603 297.026601,23.186852 298.193268,23.186852 C298.912018,23.186852 299.53181,23.327477 300.052643,23.608727 C300.583893,23.889977 300.964101,24.280602 301.193268,24.780602 C301.422435,25.280602 301.537018,26.0045603 301.537018,26.952477 L301.537018,33.077485 L299.896393,33.077485 L299.896393,26.952477 C299.896393,26.1295603 299.71931,25.530602 299.365143,25.155602 C299.010976,24.780602 298.505768,24.593102 297.849518,24.593102 C297.370351,24.593102 296.917226,24.7233103 296.490143,24.983727 C296.06306,25.233727 295.755768,25.5722687 295.568268,25.999352 C295.391185,26.4264353 295.302643,27.0201853 295.302643,27.780602 L295.302643,33.077485 L293.662018,33.077485 L293.662018,33.077485 Z M303.428375,28.233735 C303.428375,26.4420633 303.923167,25.1191441 304.91275,24.2649774 C305.746083,23.5462274 306.761708,23.1868524 307.959625,23.1868524 C309.292958,23.1868524 310.3815,23.6243524 311.22525,24.4993524 C312.069,25.3639357 312.490875,26.5670633 312.490875,28.108735 C312.490875,29.3483183 312.303375,30.327485 311.928375,31.046235 C311.553375,31.7545683 311.0065,32.3066517 310.28775,32.702485 C309.579417,33.0983183 308.803375,33.296235 307.959625,33.296235 C306.605458,33.296235 305.511708,32.8639433 304.678375,31.99936 C303.845042,31.12436 303.428375,29.8691517 303.428375,28.233735 Z M305.115875,28.233735 C305.115875,29.4733183 305.3815,30.40561 305.91275,31.03061 C306.454417,31.6451933 307.136708,31.952485 307.959625,31.952485 C308.772125,31.952485 309.449208,31.6451933 309.990875,31.03061 C310.532542,30.40561 310.803375,29.4576933 310.803375,28.18686 C310.803375,26.9889383 310.532542,26.0826857 309.990875,25.4681024 C309.449208,24.8535191 308.772125,24.5462274 307.959625,24.5462274 C307.136708,24.5462274 306.454417,24.8535191 305.91275,25.4681024 C305.3815,26.0722691 305.115875,26.9941466 305.115875,28.233735 Z M317.991577,31.608735 L318.225952,33.06186 C317.767619,33.15561 317.35616,33.202485 316.991577,33.202485 C316.397827,33.202485 315.934285,33.108735 315.600952,32.921235 C315.278035,32.733735 315.048869,32.4889433 314.913452,32.18686 C314.778035,31.87436 314.710327,31.2285267 314.710327,30.24936 L314.710327,24.6712274 L313.507202,24.6712274 L313.507202,23.4056024 L314.710327,23.4056024 L314.710327,21.0149774 L316.335327,20.0306024 L316.335327,23.4056024 L317.991577,23.4056024 L317.991577,24.6712274 L316.335327,24.6712274 L316.335327,30.34311 C316.335327,30.8014433 316.361369,31.0983183 316.413452,31.233735 C316.475952,31.3691517 316.569702,31.4785267 316.694702,31.56186 C316.830119,31.6347767 317.022827,31.671235 317.272827,31.671235 C317.44991,31.671235 317.689494,31.6504017 317.991577,31.608735 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="447.53366 10.47244 518.43127 10.47244 518.43127 41.842511 447.53366 41.842511"></polygon>
+                <path d="M471.62164,31.49936 L471.62164,33.077485 L462.793545,33.077485 C462.783129,32.6816517 462.845629,32.3014433 462.981045,31.93686 C463.210212,31.3326933 463.569587,30.7389433 464.05917,30.15561 C464.55917,29.5722767 465.27792,28.9004017 466.21542,28.139985 C467.663337,26.9524799 468.642504,26.0097691 469.15292,25.3118524 C469.663337,24.6139357 469.918545,23.9576857 469.918545,23.3431024 C469.918545,22.6868524 469.68417,22.1399774 469.21542,21.7024774 C468.757087,21.2545607 468.15292,21.0306024 467.40292,21.0306024 C466.611254,21.0306024 465.975837,21.2701857 465.49667,21.7493524 C465.02792,22.2181024 464.793545,22.8691441 464.793545,23.7024774 L463.106045,23.5306024 C463.220629,22.2701857 463.65292,21.3118524 464.40292,20.6556024 C465.163337,19.9993524 466.173754,19.6712274 467.43417,19.6712274 C468.71542,19.6712274 469.731045,20.0253941 470.481045,20.7337274 C471.231025,21.4420607 471.606015,22.3222691 471.606015,23.3743524 C471.606015,23.9056024 471.49664,24.4316441 471.27789,24.9524774 C471.05916,25.4733107 470.694587,26.0201857 470.18417,26.5931024 C469.68417,27.1556024 468.84042,27.9368549 467.65292,28.93686 C466.673754,29.7597767 466.043545,30.3222767 465.762295,30.62436 C465.481045,30.9160267 465.251879,31.2076933 465.074795,31.49936 L471.62164,31.49936 Z M473.387997,26.4837274 C473.387997,24.9108107 473.549456,23.6451857 473.872372,22.6868524 C474.195289,21.7181024 474.674456,20.9733107 475.309872,20.4524774 C475.955706,19.9316441 476.762997,19.6712274 477.731747,19.6712274 C478.450497,19.6712274 479.080706,19.8170607 479.622372,20.1087274 C480.164039,20.3899774 480.606747,20.8066441 480.950497,21.3587274 C481.304664,21.9003941 481.580706,22.5618524 481.778622,23.3431024 C481.986956,24.1139357 482.091122,25.1608107 482.091122,26.4837274 C482.091122,28.0566491 481.929664,29.327485 481.606747,30.296235 C481.283831,31.2545683 480.799456,31.99936 480.153622,32.53061 C479.518206,33.0514433 478.710914,33.31186 477.731747,33.31186 C476.450497,33.31186 475.440081,32.8483183 474.700497,31.921235 C473.825497,30.8066517 473.387997,28.9941491 473.387997,26.4837274 Z M475.059872,26.4837274 C475.059872,28.6816491 475.315081,30.1451933 475.825497,30.87436 C476.346331,31.59311 476.981747,31.952485 477.731747,31.952485 C478.492164,31.952485 479.127581,31.5879017 479.637997,30.858735 C480.148414,30.1295683 480.403622,28.6712325 480.403622,26.4837274 C480.403622,24.2858107 480.148414,22.8274774 479.637997,22.1087274 C479.127581,21.3899774 478.486956,21.0306024 477.716122,21.0306024 C476.966122,21.0306024 476.367164,21.3483107 475.919247,21.9837274 C475.346331,22.7962274 475.059872,24.2962274 475.059872,26.4837274 Z M483.763729,26.4837274 C483.763729,24.9108107 483.925188,23.6451857 484.248104,22.6868524 C484.571021,21.7181024 485.050188,20.9733107 485.685604,20.4524774 C486.331438,19.9316441 487.138729,19.6712274 488.107479,19.6712274 C488.826229,19.6712274 489.456438,19.8170607 489.998104,20.1087274 C490.539771,20.3899774 490.982479,20.8066441 491.326229,21.3587274 C491.680396,21.9003941 491.956438,22.5618524 492.154354,23.3431024 C492.362688,24.1139357 492.466854,25.1608107 492.466854,26.4837274 C492.466854,28.0566491 492.305396,29.327485 491.982479,30.296235 C491.659563,31.2545683 491.175188,31.99936 490.529354,32.53061 C489.893938,33.0514433 489.086646,33.31186 488.107479,33.31186 C486.826229,33.31186 485.815813,32.8483183 485.076229,31.921235 C484.201229,30.8066517 483.763729,28.9941491 483.763729,26.4837274 Z M485.435604,26.4837274 C485.435604,28.6816491 485.690813,30.1451933 486.201229,30.87436 C486.722063,31.59311 487.357479,31.952485 488.107479,31.952485 C488.867896,31.952485 489.503313,31.5879017 490.013729,30.858735 C490.524146,30.1295683 490.779354,28.6712325 490.779354,26.4837274 C490.779354,24.2858107 490.524146,22.8274774 490.013729,22.1087274 C489.503313,21.3899774 488.862688,21.0306024 488.091854,21.0306024 C487.341854,21.0306024 486.742896,21.3483107 486.294979,21.9837274 C485.722063,22.7962274 485.435604,24.2962274 485.435604,26.4837274 Z M494.139461,26.4837274 C494.139461,24.9108107 494.30092,23.6451857 494.623836,22.6868524 C494.946753,21.7181024 495.42592,20.9733107 496.061336,20.4524774 C496.70717,19.9316441 497.514461,19.6712274 498.483211,19.6712274 C499.201961,19.6712274 499.83217,19.8170607 500.373836,20.1087274 C500.915503,20.3899774 501.358211,20.8066441 501.701961,21.3587274 C502.056128,21.9003941 502.33217,22.5618524 502.530086,23.3431024 C502.73842,24.1139357 502.842586,25.1608107 502.842586,26.4837274 C502.842586,28.0566491 502.681128,29.327485 502.358211,30.296235 C502.035295,31.2545683 501.55092,31.99936 500.905086,32.53061 C500.26967,33.0514433 499.462378,33.31186 498.483211,33.31186 C497.201961,33.31186 496.191545,32.8483183 495.451961,31.921235 C494.576961,30.8066517 494.139461,28.9941491 494.139461,26.4837274 Z M495.811336,26.4837274 C495.811336,28.6816491 496.066545,30.1451933 496.576961,30.87436 C497.097795,31.59311 497.733211,31.952485 498.483211,31.952485 C499.243628,31.952485 499.879045,31.5879017 500.389461,30.858735 C500.899878,30.1295683 501.155086,28.6712325 501.155086,26.4837274 C501.155086,24.2858107 500.899878,22.8274774 500.389461,22.1087274 C499.879045,21.3899774 499.23842,21.0306024 498.467586,21.0306024 C497.717586,21.0306024 497.118628,21.3483107 496.670711,21.9837274 C496.097795,22.7962274 495.811336,24.2962274 495.811336,26.4837274 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polyline id="Path" fill-opacity="0" fill="#000000" points="208.80545 39.14516 208.80545 106.29134 24.80545 106.29134 24.80545 39.20815"></polyline>
+                <polyline id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round" points="208.80545 57.14516 208.80545 106.29134 24.80545 106.29134 24.80545 39.20815"></polyline>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="213.76065 57.14516 208.80545 43.530872 203.85025 57.14516"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="53.1076 90.47244 170.14696 90.47244 170.14696 121.842511 53.1076 121.842511"></polygon>
+                <path d="M67.43531,113.07748 L67.43531,99.718105 L72.482185,99.718105 C73.3676017,99.718105 74.044685,99.7597717 74.513435,99.843105 C75.1592683,99.9576883 75.7061433,100.17123 76.15406,100.48373 C76.6019767,100.785813 76.9613517,101.212897 77.232185,101.76498 C77.5030183,102.317063 77.638435,102.926438 77.638435,103.593105 C77.638435,104.728522 77.2738517,105.692063 76.544685,106.48373 C75.825935,107.26498 74.5186433,107.655605 72.62281,107.655605 L69.200935,107.655605 L69.200935,113.07748 L67.43531,113.07748 Z M69.200935,106.07748 L72.65406,106.07748 C73.7998933,106.07748 74.6123933,105.863938 75.09156,105.436855 C75.5707267,105.009772 75.81031,104.410813 75.81031,103.63998 C75.81031,103.07748 75.669685,102.598313 75.388435,102.20248 C75.107185,101.806647 74.732185,101.54623 74.263435,101.42123 C73.9717683,101.337897 73.4248933,101.29623 72.62281,101.29623 L69.200935,101.29623 L69.200935,106.07748 Z M79.066383,108.23373 C79.066383,106.442063 79.5611747,105.119147 80.550758,104.26498 C81.3840913,103.54623 82.3997163,103.186855 83.597633,103.186855 C84.9309663,103.186855 86.019508,103.624355 86.863258,104.499355 C87.7070029,105.363938 88.1288754,106.567063 88.1288754,108.10873 C88.1288754,109.348313 87.9413754,110.32748 87.566383,111.04623 C87.1913754,111.754563 86.6445029,112.306647 85.925758,112.70248 C85.2174247,113.098313 84.441383,113.29623 83.597633,113.29623 C82.2434663,113.29623 81.1497163,112.863938 80.316383,111.999355 C79.4830497,111.124355 79.066383,109.869147 79.066383,108.23373 Z M80.753883,108.23373 C80.753883,109.473313 81.019508,110.405605 81.550758,111.030605 C82.0924247,111.645188 82.7747163,111.95248 83.597633,111.95248 C84.410133,111.95248 85.0872163,111.645188 85.628883,111.030605 C86.1705497,110.405605 86.441383,109.457688 86.441383,108.186855 C86.441383,106.988938 86.1705497,106.082688 85.628883,105.468105 C85.0872163,104.853522 84.410133,104.54623 83.597633,104.54623 C82.7747163,104.54623 82.0924247,104.853522 81.550758,105.468105 C81.019508,106.072272 80.753883,106.994147 80.753883,108.23373 Z M91.832718,113.07748 L88.863968,103.405605 L90.567093,103.405605 L92.098343,108.98373 L92.676468,111.061855 C92.707718,110.957688 92.8743847,110.291022 93.176468,109.061855 L94.723343,103.405605 L96.410843,103.405605 L97.848343,109.01498 L98.332718,110.85873 L98.895218,108.999355 L100.551468,103.405605 L102.145218,103.405605 L99.113968,113.07748 L97.410843,113.07748 L95.879593,107.280605 L95.504593,105.63998 L93.551468,113.07748 L91.832718,113.07748 Z M110.149414,109.968105 L111.836914,110.17123 C111.566081,111.160813 111.071289,111.931647 110.352539,112.48373 C109.633789,113.025397 108.711914,113.29623 107.586914,113.29623 C106.170247,113.29623 105.045247,112.863938 104.211914,111.999355 C103.388997,111.124355 102.977539,109.900397 102.977539,108.32748 C102.977539,106.692063 103.394206,105.426438 104.227539,104.530605 C105.071289,103.634772 106.159831,103.186855 107.493164,103.186855 C108.784831,103.186855 109.836914,103.629563 110.649414,104.51498 C111.472331,105.38998 111.883789,106.624355 111.883789,108.218105 C111.883789,108.322272 111.883789,108.468105 111.883789,108.655605 L104.665039,108.655605 C104.727539,109.718105 105.029622,110.535813 105.571289,111.10873 C106.112956,111.67123 106.784831,111.95248 107.586914,111.95248 C108.191081,111.95248 108.706706,111.79623 109.133789,111.48373 C109.560872,111.160813 109.899414,110.655605 110.149414,109.968105 Z M104.758789,107.311855 L110.165039,107.311855 C110.092122,106.499355 109.883789,105.88998 109.540039,105.48373 C109.019206,104.848313 108.342122,104.530605 107.508789,104.530605 C106.758789,104.530605 106.123372,104.785813 105.602539,105.29623 C105.092122,105.79623 104.810872,106.468105 104.758789,107.311855 Z M113.884506,113.07748 L113.884506,103.405605 L115.353256,103.405605 L115.353256,104.874355 C115.728256,104.186855 116.072006,103.73373 116.384506,103.51498 C116.707423,103.29623 117.061589,103.186855 117.447006,103.186855 C117.999089,103.186855 118.561589,103.363938 119.134506,103.718105 L118.572006,105.23373 C118.165756,104.994147 117.764714,104.874355 117.368881,104.874355 C117.004298,104.874355 116.681381,104.98373 116.400131,105.20248 C116.118881,105.42123 115.915756,105.718105 115.790756,106.093105 C115.603256,106.676438 115.509506,107.317063 115.509506,108.01498 L115.509506,113.07748 L113.884506,113.07748 Z M119.722183,108.780605 L121.378433,108.63998 C121.461766,109.306647 121.644058,109.853522 121.925308,110.280605 C122.216975,110.707688 122.664891,111.056647 123.269058,111.32748 C123.883641,111.587897 124.571141,111.718105 125.331558,111.718105 C125.998225,111.718105 126.591975,111.619147 127.112808,111.42123 C127.633641,111.223313 128.019058,110.95248 128.269058,110.60873 C128.519058,110.254563 128.644058,109.869147 128.644058,109.45248 C128.644058,109.035813 128.519058,108.67123 128.269058,108.35873 C128.029475,108.04623 127.633641,107.780605 127.081558,107.561855 C126.716975,107.426438 125.9201,107.212897 124.690933,106.92123 C123.472183,106.619147 122.618016,106.337897 122.128433,106.07748 C121.4826,105.744147 121.003433,105.332688 120.690933,104.843105 C120.378433,104.343105 120.222183,103.785813 120.222183,103.17123 C120.222183,102.504563 120.414891,101.879563 120.800308,101.29623 C121.185725,100.70248 121.743016,100.254563 122.472183,99.95248 C123.211766,99.6503967 124.029475,99.499355 124.925308,99.499355 C125.914891,99.499355 126.784683,99.6608133 127.534683,99.98373 C128.2951,100.29623 128.878433,100.76498 129.284683,101.38998 C129.690933,102.004563 129.909683,102.70248 129.940933,103.48373 L128.253433,103.60873 C128.159683,102.76498 127.847183,102.129563 127.315933,101.70248 C126.7951,101.26498 126.024266,101.04623 125.003433,101.04623 C123.930516,101.04623 123.149266,101.244147 122.659683,101.63998 C122.1701,102.035813 121.925308,102.509772 121.925308,103.061855 C121.925308,103.541022 122.102391,103.931647 122.456558,104.23373 C122.789891,104.54623 123.675308,104.869147 125.112808,105.20248 C126.550308,105.525397 127.534683,105.806647 128.065933,106.04623 C128.847183,106.400397 129.4201,106.853522 129.784683,107.405605 C130.159683,107.957688 130.347183,108.593105 130.347183,109.311855 C130.347183,110.020188 130.144058,110.692063 129.737808,111.32748 C129.331558,111.95248 128.748225,112.442063 127.987808,112.79623 C127.227391,113.13998 126.368016,113.311855 125.409683,113.311855 C124.20135,113.311855 123.185725,113.134772 122.362808,112.780605 C121.550308,112.426438 120.909683,111.895188 120.440933,111.186855 C119.9826,110.478522 119.743016,109.676438 119.722183,108.780605 Z M132.556381,113.07748 L132.556381,99.718105 L134.197006,99.718105 L134.197006,104.51498 C134.957423,103.629563 135.920964,103.186855 137.087631,103.186855 C137.806381,103.186855 138.426173,103.32748 138.947006,103.60873 C139.478256,103.88998 139.858464,104.280605 140.087631,104.780605 C140.316798,105.280605 140.431381,106.004563 140.431381,106.95248 L140.431381,113.07748 L138.790756,113.07748 L138.790756,106.95248 C138.790756,106.129563 138.613673,105.530605 138.259506,105.155605 C137.905339,104.780605 137.400131,104.593105 136.743881,104.593105 C136.264714,104.593105 135.811589,104.723313 135.384506,104.98373 C134.957423,105.23373 134.650131,105.572272 134.462631,105.999355 C134.285548,106.426438 134.197006,107.020188 134.197006,107.780605 L134.197006,113.07748 L132.556381,113.07748 Z M142.322723,108.23373 C142.322723,106.442063 142.817515,105.119147 143.807098,104.26498 C144.640431,103.54623 145.656056,103.186855 146.853973,103.186855 C148.187306,103.186855 149.275848,103.624355 150.119598,104.499355 C150.963348,105.363938 151.385223,106.567063 151.385223,108.10873 C151.385223,109.348313 151.197723,110.32748 150.822723,111.04623 C150.447723,111.754563 149.900848,112.306647 149.182098,112.70248 C148.473765,113.098313 147.697723,113.29623 146.853973,113.29623 C145.499806,113.29623 144.406056,112.863938 143.572723,111.999355 C142.73939,111.124355 142.322723,109.869147 142.322723,108.23373 Z M144.010223,108.23373 C144.010223,109.473313 144.275848,110.405605 144.807098,111.030605 C145.348765,111.645188 146.031056,111.95248 146.853973,111.95248 C147.666473,111.95248 148.343556,111.645188 148.885223,111.030605 C149.42689,110.405605 149.697723,109.457688 149.697723,108.186855 C149.697723,106.988938 149.42689,106.082688 148.885223,105.468105 C148.343556,104.853522 147.666473,104.54623 146.853973,104.54623 C146.031056,104.54623 145.348765,104.853522 144.807098,105.468105 C144.275848,106.072272 144.010223,106.994147 144.010223,108.23373 Z M156.88594,111.60873 L157.120315,113.061855 C156.661982,113.155605 156.250523,113.20248 155.88594,113.20248 C155.29219,113.20248 154.828648,113.10873 154.495315,112.92123 C154.172398,112.73373 153.943232,112.488938 153.807815,112.186855 C153.672398,111.874355 153.60469,111.228522 153.60469,110.249355 L153.60469,104.67123 L152.401565,104.67123 L152.401565,103.405605 L153.60469,103.405605 L153.60469,101.01498 L155.22969,100.030605 L155.22969,103.405605 L156.88594,103.405605 L156.88594,104.67123 L155.22969,104.67123 L155.22969,110.343105 C155.22969,110.801438 155.255732,111.098313 155.307815,111.23373 C155.370315,111.369147 155.464065,111.478522 155.589065,111.561855 C155.724482,111.634772 155.91719,111.67123 156.16719,111.67123 C156.344273,111.67123 156.583857,111.650397 156.88594,111.60873 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-wdg.svg b/docs/reference/images/analysis/token-graph-wdg.svg
new file mode 100644
index 0000000000000..992637bd668d5
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-wdg.svg
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="602px" height="123px" viewBox="0 0 602 123" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Analysis-(2)" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
+                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
+                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="15.088913 39.14516 34.52198 39.14516 34.52198 60.971927 15.088913 60.971927"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="199.08891 39.14516 218.52197 39.14516 218.52197 60.971927 199.08891 60.971927"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="73.96063 10.47244 152.35432 10.47244 152.35432 41.842511 73.96063 41.842511"></polygon>
+                <path d="M84.39813,33.07748 L84.39813,19.718097 L89.445005,19.718097 C90.3304217,19.718097 91.007505,19.7597637 91.476255,19.843097 C92.1220883,19.9576803 92.6689633,20.171222 93.11688,20.483722 C93.5647967,20.7858053 93.9241717,21.2128887 94.195005,21.764972 C94.4658383,22.3170553 94.601255,22.9264303 94.601255,23.593097 C94.601255,24.7285137 94.2366717,25.6920553 93.507505,26.483722 C92.788755,27.264972 91.4814633,27.655597 89.58563,27.655597 L86.163755,27.655597 L86.163755,33.07748 L84.39813,33.07748 L84.39813,33.07748 Z M86.163755,26.0774724 L89.61688,26.0774724 C90.7627133,26.0774724 91.5752133,25.8639307 92.05438,25.4368474 C92.5335467,25.0097641 92.77313,24.4108057 92.77313,23.6399724 C92.77313,23.0774724 92.632505,22.5983057 92.351255,22.2024724 C92.070005,21.8066391 91.695005,21.5462224 91.226255,21.4212224 C90.9345883,21.3378891 90.3877133,21.2962224 89.58563,21.2962224 L86.163755,21.2962224 L86.163755,26.0774724 L86.163755,26.0774724 Z M96.029203,28.23373 C96.029203,26.4420583 96.5239947,25.1191391 97.513578,24.2649724 C98.3469113,23.5462224 99.3625363,23.1868474 100.560453,23.1868474 C101.893786,23.1868474 102.982328,23.6243474 103.826078,24.4993474 C104.669828,25.3639307 105.091703,26.5670583 105.091703,28.10873 C105.091703,29.3483133 104.904203,30.32748 104.529203,31.04623 C104.154203,31.7545633 103.607328,32.3066467 102.888578,32.70248 C102.180245,33.0983133 101.404203,33.29623 100.560453,33.29623 C99.2062863,33.29623 98.1125363,32.8639383 97.279203,31.999355 C96.4458697,31.124355 96.029203,29.8691467 96.029203,28.23373 Z M97.716703,28.23373 C97.716703,29.4733133 97.982328,30.405605 98.513578,31.030605 C99.0552447,31.6451883 99.7375363,31.95248 100.560453,31.95248 C101.372953,31.95248 102.050036,31.6451883 102.591703,31.030605 C103.13337,30.405605 103.404203,29.4576883 103.404203,28.186855 C103.404203,26.9889333 103.13337,26.0826807 102.591703,25.4680974 C102.050036,24.8535141 101.372953,24.5462224 100.560453,24.5462224 C99.7375363,24.5462224 99.0552447,24.8535141 98.513578,25.4680974 C97.982328,26.0722641 97.716703,26.9941416 97.716703,28.23373 Z M108.795545,33.07748 L105.826795,23.405597 L107.52992,23.405597 L109.06117,28.9837296 L109.639295,31.0618546 C109.670545,30.9576879 109.837212,30.2910213 110.139295,29.0618546 L111.68617,23.405597 L113.37367,23.405597 L114.81117,29.0149796 L115.295545,30.8587296 L115.858045,28.9993546 L117.514295,23.405597 L119.108045,23.405597 L116.076795,33.07748 L114.37367,33.07748 L112.84242,27.2805974 L112.46742,25.6399724 L110.514295,33.07748 L108.795545,33.07748 Z M127.112241,29.968105 L128.799741,30.17123 C128.528908,31.1608133 128.034116,31.9316467 127.315366,32.48373 C126.596616,33.0253967 125.674741,33.29623 124.549741,33.29623 C123.133074,33.29623 122.008074,32.8639383 121.174741,31.999355 C120.351824,31.124355 119.940366,29.9003967 119.940366,28.32748 C119.940366,26.6920583 120.357033,25.4264307 121.190366,24.5305974 C122.034116,23.6347641 123.122658,23.1868474 124.455991,23.1868474 C125.747658,23.1868474 126.799741,23.6295557 127.612241,24.5149724 C128.435158,25.3899724 128.846616,26.6243499 128.846616,28.218105 C128.846616,28.3222717 128.846616,28.468105 128.846616,28.655605 L121.627866,28.655605 C121.690366,29.718105 121.992449,30.5358133 122.534116,31.10873 C123.075783,31.67123 123.747658,31.95248 124.549741,31.95248 C125.153908,31.95248 125.669533,31.79623 126.096616,31.48373 C126.523699,31.1608133 126.862241,30.655605 127.112241,29.968105 Z M121.721616,27.3118474 L127.127866,27.3118474 C127.054949,26.4993474 126.846616,25.8899724 126.502866,25.4837224 C125.982033,24.8483057 125.304949,24.5305974 124.471616,24.5305974 C123.721616,24.5305974 123.086199,24.7858057 122.565366,25.2962224 C122.054949,25.7962224 121.773699,26.4680974 121.721616,27.3118474 Z M130.847333,33.07748 L130.847333,23.405597 L132.316083,23.405597 L132.316083,24.874347 C132.691083,24.186847 133.034833,23.733722 133.347333,23.514972 C133.67025,23.296222 134.024416,23.186847 134.409833,23.186847 C134.961916,23.186847 135.524416,23.3639303 136.097333,23.718097 L135.534833,25.233722 C135.128583,24.9941387 134.727541,24.874347 134.331708,24.874347 C133.967125,24.874347 133.644208,24.983722 133.362958,25.202472 C133.081708,25.421222 132.878583,25.718097 132.753583,26.093097 C132.566083,26.6764303 132.472333,27.3170579 132.472333,28.0149796 L132.472333,33.07748 L130.847333,33.07748 L130.847333,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="265.96063 10.47244 332.38583 10.47244 332.38583 41.842511 265.96063 41.842511"></polygon>
+                <path d="M280.82782,28.78061 L282.48407,28.639985 C282.567403,29.3066517 282.749695,29.8535267 283.030945,30.28061 C283.322612,30.7076933 283.770528,31.0566517 284.374695,31.327485 C284.989278,31.5879017 285.676778,31.71811 286.437195,31.71811 C287.103862,31.71811 287.697612,31.6191517 288.218445,31.421235 C288.739278,31.2233183 289.124695,30.952485 289.374695,30.608735 C289.624695,30.2545683 289.749695,29.8691517 289.749695,29.452485 C289.749695,29.0358183 289.624695,28.671235 289.374695,28.358735 C289.135112,28.0462299 288.739278,27.7806024 288.187195,27.5618524 C287.822612,27.4264357 287.025737,27.2128941 285.79657,26.9212274 C284.57782,26.6191441 283.723653,26.3378941 283.23407,26.0774774 C282.588237,25.7441441 282.10907,25.3326857 281.79657,24.8431024 C281.48407,24.3431024 281.32782,23.7858107 281.32782,23.1712274 C281.32782,22.5045607 281.520528,21.8795607 281.905945,21.2962274 C282.291362,20.7024774 282.848653,20.2545607 283.57782,19.9524774 C284.317403,19.6503941 285.135112,19.4993524 286.030945,19.4993524 C287.020528,19.4993524 287.89032,19.6608107 288.64032,19.9837274 C289.400737,20.2962274 289.98407,20.7649774 290.39032,21.3899774 C290.79657,22.0045607 291.01532,22.7024774 291.04657,23.4837274 L289.35907,23.6087274 C289.26532,22.7649774 288.95282,22.1295607 288.42157,21.7024774 C287.900737,21.2649774 287.129903,21.0462274 286.10907,21.0462274 C285.036153,21.0462274 284.254903,21.2441441 283.76532,21.6399774 C283.275737,22.0358107 283.030945,22.5097691 283.030945,23.0618524 C283.030945,23.5410191 283.208028,23.9316441 283.562195,24.2337274 C283.895528,24.5462274 284.780945,24.8691441 286.218445,25.2024774 C287.655945,25.5253941 288.64032,25.8066441 289.17157,26.0462274 C289.95282,26.4003941 290.525737,26.8535191 290.89032,27.4056024 C291.26532,27.9576908 291.45282,28.59311 291.45282,29.31186 C291.45282,30.0201933 291.249695,30.6920683 290.843445,31.327485 C290.437195,31.952485 289.853862,32.4420683 289.093445,32.796235 C288.333028,33.139985 287.473653,33.31186 286.51532,33.31186 C285.306987,33.31186 284.291362,33.1347767 283.468445,32.78061 C282.655945,32.4264433 282.01532,31.8951933 281.54657,31.18686 C281.088237,30.4785267 280.848653,29.6764433 280.82782,28.78061 Z M293.662018,33.077485 L293.662018,19.718102 L295.302643,19.718102 L295.302643,24.514977 C296.06306,23.6295603 297.026601,23.186852 298.193268,23.186852 C298.912018,23.186852 299.53181,23.327477 300.052643,23.608727 C300.583893,23.889977 300.964101,24.280602 301.193268,24.780602 C301.422435,25.280602 301.537018,26.0045603 301.537018,26.952477 L301.537018,33.077485 L299.896393,33.077485 L299.896393,26.952477 C299.896393,26.1295603 299.71931,25.530602 299.365143,25.155602 C299.010976,24.780602 298.505768,24.593102 297.849518,24.593102 C297.370351,24.593102 296.917226,24.7233103 296.490143,24.983727 C296.06306,25.233727 295.755768,25.5722687 295.568268,25.999352 C295.391185,26.4264353 295.302643,27.0201853 295.302643,27.780602 L295.302643,33.077485 L293.662018,33.077485 L293.662018,33.077485 Z M303.428375,28.233735 C303.428375,26.4420633 303.923167,25.1191441 304.91275,24.2649774 C305.746083,23.5462274 306.761708,23.1868524 307.959625,23.1868524 C309.292958,23.1868524 310.3815,23.6243524 311.22525,24.4993524 C312.069,25.3639357 312.490875,26.5670633 312.490875,28.108735 C312.490875,29.3483183 312.303375,30.327485 311.928375,31.046235 C311.553375,31.7545683 311.0065,32.3066517 310.28775,32.702485 C309.579417,33.0983183 308.803375,33.296235 307.959625,33.296235 C306.605458,33.296235 305.511708,32.8639433 304.678375,31.99936 C303.845042,31.12436 303.428375,29.8691517 303.428375,28.233735 Z M305.115875,28.233735 C305.115875,29.4733183 305.3815,30.40561 305.91275,31.03061 C306.454417,31.6451933 307.136708,31.952485 307.959625,31.952485 C308.772125,31.952485 309.449208,31.6451933 309.990875,31.03061 C310.532542,30.40561 310.803375,29.4576933 310.803375,28.18686 C310.803375,26.9889383 310.532542,26.0826857 309.990875,25.4681024 C309.449208,24.8535191 308.772125,24.5462274 307.959625,24.5462274 C307.136708,24.5462274 306.454417,24.8535191 305.91275,25.4681024 C305.3815,26.0722691 305.115875,26.9941466 305.115875,28.233735 Z M317.991577,31.608735 L318.225952,33.06186 C317.767619,33.15561 317.35616,33.202485 316.991577,33.202485 C316.397827,33.202485 315.934285,33.108735 315.600952,32.921235 C315.278035,32.733735 315.048869,32.4889433 314.913452,32.18686 C314.778035,31.87436 314.710327,31.2285267 314.710327,30.24936 L314.710327,24.6712274 L313.507202,24.6712274 L313.507202,23.4056024 L314.710327,23.4056024 L314.710327,21.0149774 L316.335327,20.0306024 L316.335327,23.4056024 L317.991577,23.4056024 L317.991577,24.6712274 L316.335327,24.6712274 L316.335327,30.34311 C316.335327,30.8014433 316.361369,31.0983183 316.413452,31.233735 C316.475952,31.3691517 316.569702,31.4785267 316.694702,31.56186 C316.830119,31.6347767 317.022827,31.671235 317.272827,31.671235 C317.44991,31.671235 317.689494,31.6504017 317.991577,31.608735 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="447.53366 10.47244 518.43127 10.47244 518.43127 41.842511 447.53366 41.842511"></polygon>
+                <path d="M471.62164,31.49936 L471.62164,33.077485 L462.793545,33.077485 C462.783129,32.6816517 462.845629,32.3014433 462.981045,31.93686 C463.210212,31.3326933 463.569587,30.7389433 464.05917,30.15561 C464.55917,29.5722767 465.27792,28.9004017 466.21542,28.139985 C467.663337,26.9524799 468.642504,26.0097691 469.15292,25.3118524 C469.663337,24.6139357 469.918545,23.9576857 469.918545,23.3431024 C469.918545,22.6868524 469.68417,22.1399774 469.21542,21.7024774 C468.757087,21.2545607 468.15292,21.0306024 467.40292,21.0306024 C466.611254,21.0306024 465.975837,21.2701857 465.49667,21.7493524 C465.02792,22.2181024 464.793545,22.8691441 464.793545,23.7024774 L463.106045,23.5306024 C463.220629,22.2701857 463.65292,21.3118524 464.40292,20.6556024 C465.163337,19.9993524 466.173754,19.6712274 467.43417,19.6712274 C468.71542,19.6712274 469.731045,20.0253941 470.481045,20.7337274 C471.231025,21.4420607 471.606015,22.3222691 471.606015,23.3743524 C471.606015,23.9056024 471.49664,24.4316441 471.27789,24.9524774 C471.05916,25.4733107 470.694587,26.0201857 470.18417,26.5931024 C469.68417,27.1556024 468.84042,27.9368549 467.65292,28.93686 C466.673754,29.7597767 466.043545,30.3222767 465.762295,30.62436 C465.481045,30.9160267 465.251879,31.2076933 465.074795,31.49936 L471.62164,31.49936 Z M473.387997,26.4837274 C473.387997,24.9108107 473.549456,23.6451857 473.872372,22.6868524 C474.195289,21.7181024 474.674456,20.9733107 475.309872,20.4524774 C475.955706,19.9316441 476.762997,19.6712274 477.731747,19.6712274 C478.450497,19.6712274 479.080706,19.8170607 479.622372,20.1087274 C480.164039,20.3899774 480.606747,20.8066441 480.950497,21.3587274 C481.304664,21.9003941 481.580706,22.5618524 481.778622,23.3431024 C481.986956,24.1139357 482.091122,25.1608107 482.091122,26.4837274 C482.091122,28.0566491 481.929664,29.327485 481.606747,30.296235 C481.283831,31.2545683 480.799456,31.99936 480.153622,32.53061 C479.518206,33.0514433 478.710914,33.31186 477.731747,33.31186 C476.450497,33.31186 475.440081,32.8483183 474.700497,31.921235 C473.825497,30.8066517 473.387997,28.9941491 473.387997,26.4837274 Z M475.059872,26.4837274 C475.059872,28.6816491 475.315081,30.1451933 475.825497,30.87436 C476.346331,31.59311 476.981747,31.952485 477.731747,31.952485 C478.492164,31.952485 479.127581,31.5879017 479.637997,30.858735 C480.148414,30.1295683 480.403622,28.6712325 480.403622,26.4837274 C480.403622,24.2858107 480.148414,22.8274774 479.637997,22.1087274 C479.127581,21.3899774 478.486956,21.0306024 477.716122,21.0306024 C476.966122,21.0306024 476.367164,21.3483107 475.919247,21.9837274 C475.346331,22.7962274 475.059872,24.2962274 475.059872,26.4837274 Z M483.763729,26.4837274 C483.763729,24.9108107 483.925188,23.6451857 484.248104,22.6868524 C484.571021,21.7181024 485.050188,20.9733107 485.685604,20.4524774 C486.331438,19.9316441 487.138729,19.6712274 488.107479,19.6712274 C488.826229,19.6712274 489.456438,19.8170607 489.998104,20.1087274 C490.539771,20.3899774 490.982479,20.8066441 491.326229,21.3587274 C491.680396,21.9003941 491.956438,22.5618524 492.154354,23.3431024 C492.362688,24.1139357 492.466854,25.1608107 492.466854,26.4837274 C492.466854,28.0566491 492.305396,29.327485 491.982479,30.296235 C491.659563,31.2545683 491.175188,31.99936 490.529354,32.53061 C489.893938,33.0514433 489.086646,33.31186 488.107479,33.31186 C486.826229,33.31186 485.815813,32.8483183 485.076229,31.921235 C484.201229,30.8066517 483.763729,28.9941491 483.763729,26.4837274 Z M485.435604,26.4837274 C485.435604,28.6816491 485.690813,30.1451933 486.201229,30.87436 C486.722063,31.59311 487.357479,31.952485 488.107479,31.952485 C488.867896,31.952485 489.503313,31.5879017 490.013729,30.858735 C490.524146,30.1295683 490.779354,28.6712325 490.779354,26.4837274 C490.779354,24.2858107 490.524146,22.8274774 490.013729,22.1087274 C489.503313,21.3899774 488.862688,21.0306024 488.091854,21.0306024 C487.341854,21.0306024 486.742896,21.3483107 486.294979,21.9837274 C485.722063,22.7962274 485.435604,24.2962274 485.435604,26.4837274 Z M494.139461,26.4837274 C494.139461,24.9108107 494.30092,23.6451857 494.623836,22.6868524 C494.946753,21.7181024 495.42592,20.9733107 496.061336,20.4524774 C496.70717,19.9316441 497.514461,19.6712274 498.483211,19.6712274 C499.201961,19.6712274 499.83217,19.8170607 500.373836,20.1087274 C500.915503,20.3899774 501.358211,20.8066441 501.701961,21.3587274 C502.056128,21.9003941 502.33217,22.5618524 502.530086,23.3431024 C502.73842,24.1139357 502.842586,25.1608107 502.842586,26.4837274 C502.842586,28.0566491 502.681128,29.327485 502.358211,30.296235 C502.035295,31.2545683 501.55092,31.99936 500.905086,32.53061 C500.26967,33.0514433 499.462378,33.31186 498.483211,33.31186 C497.201961,33.31186 496.191545,32.8483183 495.451961,31.921235 C494.576961,30.8066517 494.139461,28.9941491 494.139461,26.4837274 Z M495.811336,26.4837274 C495.811336,28.6816491 496.066545,30.1451933 496.576961,30.87436 C497.097795,31.59311 497.733211,31.952485 498.483211,31.952485 C499.243628,31.952485 499.879045,31.5879017 500.389461,30.858735 C500.899878,30.1295683 501.155086,28.6712325 501.155086,26.4837274 C501.155086,24.2858107 500.899878,22.8274774 500.389461,22.1087274 C499.879045,21.3899774 499.23842,21.0306024 498.467586,21.0306024 C497.717586,21.0306024 497.118628,21.3483107 496.670711,21.9837274 C496.097795,22.7962274 495.811336,24.2962274 495.811336,26.4837274 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="380.66196 40.35864 400.095035 40.35864 400.095035 62.185407 380.66196 62.185407"></polygon>
+                <polyline id="Path" fill-opacity="0" fill="#000000" points="390.37848 40.35864 390.37848 106.29133 24.80368 106.29133 24.80368 39.13029"></polyline>
+                <polyline id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round" points="390.37848 58.35864 390.37848 106.291333 24.80368 106.291333 24.80368 39.130293"></polyline>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="395.33368 58.35864 390.37848 44.744336 385.42328 58.35864"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="149.10759 90.47244 266.14697 90.47244 266.14697 121.842511 149.10759 121.842511"></polygon>
+                <path d="M163.4353,113.07748 L163.4353,99.718105 L168.482175,99.718105 C169.367592,99.718105 170.044675,99.7597717 170.513425,99.843105 C171.159258,99.9576883 171.706133,100.17123 172.15405,100.48373 C172.601967,100.785813 172.961342,101.212897 173.232175,101.76498 C173.503008,102.317063 173.638425,102.926438 173.638425,103.593105 C173.638425,104.728522 173.273842,105.692063 172.544675,106.48373 C171.825925,107.26498 170.518633,107.655605 168.6228,107.655605 L165.200925,107.655605 L165.200925,113.07748 L163.4353,113.07748 Z M165.200925,106.07748 L168.65405,106.07748 C169.799883,106.07748 170.612383,105.863938 171.09155,105.436855 C171.570717,105.009772 171.8103,104.410813 171.8103,103.63998 C171.8103,103.07748 171.669675,102.598313 171.388425,102.20248 C171.107175,101.806647 170.732175,101.54623 170.263425,101.42123 C169.971758,101.337897 169.424883,101.29623 168.6228,101.29623 L165.200925,101.29623 L165.200925,106.07748 Z M175.066373,108.23373 C175.066373,106.442063 175.561165,105.119147 176.550748,104.26498 C177.384081,103.54623 178.399706,103.186855 179.597623,103.186855 C180.930956,103.186855 182.019498,103.624355 182.863248,104.499355 C183.706998,105.363938 184.128873,106.567063 184.128873,108.10873 C184.128873,109.348313 183.941373,110.32748 183.566373,111.04623 C183.191373,111.754563 182.644498,112.306647 181.925748,112.70248 C181.217415,113.098313 180.441373,113.29623 179.597623,113.29623 C178.243456,113.29623 177.149706,112.863938 176.316373,111.999355 C175.48304,111.124355 175.066373,109.869147 175.066373,108.23373 Z M176.753873,108.23373 C176.753873,109.473313 177.019498,110.405605 177.550748,111.030605 C178.092415,111.645188 178.774706,111.95248 179.597623,111.95248 C180.410123,111.95248 181.087206,111.645188 181.628873,111.030605 C182.17054,110.405605 182.441373,109.457688 182.441373,108.186855 C182.441373,106.988938 182.17054,106.082688 181.628873,105.468105 C181.087206,104.853522 180.410123,104.54623 179.597623,104.54623 C178.774706,104.54623 178.092415,104.853522 177.550748,105.468105 C177.019498,106.072272 176.753873,106.994147 176.753873,108.23373 Z M187.832715,113.07748 L184.863965,103.405605 L186.56709,103.405605 L188.09834,108.98373 L188.676465,111.061855 C188.707715,110.957688 188.874382,110.291022 189.176465,109.061855 L190.72334,103.405605 L192.41084,103.405605 L193.84834,109.01498 L194.332715,110.85873 L194.895215,108.999355 L196.551465,103.405605 L198.145215,103.405605 L195.113965,113.07748 L193.41084,113.07748 L191.87959,107.280605 L191.50459,105.63998 L189.551465,113.07748 L187.832715,113.07748 Z M206.149411,109.968105 L207.836911,110.17123 C207.566078,111.160813 207.071286,111.931647 206.352536,112.48373 C205.633786,113.025397 204.711911,113.29623 203.586911,113.29623 C202.170244,113.29623 201.045244,112.863938 200.211911,111.999355 C199.388994,111.124355 198.977536,109.900397 198.977536,108.32748 C198.977536,106.692063 199.394203,105.426438 200.227536,104.530605 C201.071286,103.634772 202.159828,103.186855 203.493161,103.186855 C204.784828,103.186855 205.836911,103.629563 206.649411,104.51498 C207.472328,105.38998 207.883786,106.624355 207.883786,108.218105 C207.883786,108.322272 207.883786,108.468105 207.883786,108.655605 L200.665036,108.655605 C200.727536,109.718105 201.029619,110.535813 201.571286,111.10873 C202.112953,111.67123 202.784828,111.95248 203.586911,111.95248 C204.191078,111.95248 204.706703,111.79623 205.133786,111.48373 C205.560869,111.160813 205.899411,110.655605 206.149411,109.968105 Z M200.758786,107.311855 L206.165036,107.311855 C206.092119,106.499355 205.883786,105.88998 205.540036,105.48373 C205.019203,104.848313 204.342119,104.530605 203.508786,104.530605 C202.758786,104.530605 202.123369,104.785813 201.602536,105.29623 C201.092119,105.79623 200.810869,106.468105 200.758786,107.311855 Z M209.884503,113.07748 L209.884503,103.405605 L211.353253,103.405605 L211.353253,104.874355 C211.728253,104.186855 212.072003,103.73373 212.384503,103.51498 C212.70742,103.29623 213.061586,103.186855 213.447003,103.186855 C213.999086,103.186855 214.561591,103.363938 215.134518,103.718105 L214.572003,105.23373 C214.165753,104.994147 213.764711,104.874355 213.368878,104.874355 C213.004295,104.874355 212.681378,104.98373 212.400128,105.20248 C212.118878,105.42123 211.915753,105.718105 211.790753,106.093105 C211.603253,106.676438 211.509503,107.317063 211.509503,108.01498 L211.509503,113.07748 L209.884503,113.07748 L209.884503,113.07748 Z M215.722195,108.780605 L217.378445,108.63998 C217.461779,109.306647 217.64407,109.853522 217.92532,110.280605 C218.216987,110.707688 218.664904,111.056647 219.26907,111.32748 C219.883654,111.587897 220.571154,111.718105 221.33157,111.718105 C221.998237,111.718105 222.591987,111.619147 223.11282,111.42123 C223.633654,111.223313 224.01907,110.95248 224.26907,110.60873 C224.51907,110.254563 224.64407,109.869147 224.64407,109.45248 C224.64407,109.035813 224.51907,108.67123 224.26907,108.35873 C224.029487,108.04623 223.633654,107.780605 223.08157,107.561855 C222.716987,107.426438 221.920112,107.212897 220.690945,106.92123 C219.472195,106.619147 218.618029,106.337897 218.128445,106.07748 C217.482612,105.744147 217.003445,105.332688 216.690945,104.843105 C216.378445,104.343105 216.222195,103.785813 216.222195,103.17123 C216.222195,102.504563 216.414904,101.879563 216.80032,101.29623 C217.185737,100.70248 217.743029,100.254563 218.472195,99.95248 C219.211779,99.6503967 220.029487,99.499355 220.92532,99.499355 C221.914904,99.499355 222.784695,99.6608133 223.534695,99.98373 C224.295112,100.29623 224.878445,100.76498 225.284695,101.38998 C225.690945,102.004563 225.909695,102.70248 225.940945,103.48373 L224.253445,103.60873 C224.159695,102.76498 223.847195,102.129563 223.315945,101.70248 C222.795112,101.26498 222.024279,101.04623 221.003445,101.04623 C219.930529,101.04623 219.149279,101.244147 218.659695,101.63998 C218.170112,102.035813 217.92532,102.509772 217.92532,103.061855 C217.92532,103.541022 218.102404,103.931647 218.45657,104.23373 C218.789904,104.54623 219.67532,104.869147 221.11282,105.20248 C222.55032,105.525397 223.534695,105.806647 224.065945,106.04623 C224.847195,106.400397 225.420112,106.853522 225.784695,107.405605 C226.159695,107.957688 226.347195,108.593105 226.347195,109.311855 C226.347195,110.020188 226.14407,110.692063 225.73782,111.32748 C225.33157,111.95248 224.748237,112.442063 223.98782,112.79623 C223.227404,113.13998 222.368029,113.311855 221.409695,113.311855 C220.201362,113.311855 219.185737,113.134772 218.36282,112.780605 C217.55032,112.426438 216.909695,111.895188 216.440945,111.186855 C215.982612,110.478522 215.743029,109.676438 215.722195,108.780605 Z M228.556393,113.07748 L228.556393,99.718105 L230.197018,99.718105 L230.197018,104.51498 C230.957435,103.629563 231.920977,103.186855 233.087643,103.186855 C233.806393,103.186855 234.426185,103.32748 234.947018,103.60873 C235.478268,103.88998 235.858477,104.280605 236.087643,104.780605 C236.31681,105.280605 236.431393,106.004563 236.431393,106.95248 L236.431393,113.07748 L234.790768,113.07748 L234.790768,106.95248 C234.790768,106.129563 234.613685,105.530605 234.259518,105.155605 C233.905352,104.780605 233.400143,104.593105 232.743893,104.593105 C232.264727,104.593105 231.811602,104.723313 231.384518,104.98373 C230.957435,105.23373 230.650143,105.572272 230.462643,105.999355 C230.28556,106.426438 230.197018,107.020188 230.197018,107.780605 L230.197018,113.07748 L228.556393,113.07748 Z M238.32272,108.23373 C238.32272,106.442063 238.817512,105.119147 239.807095,104.26498 C240.640429,103.54623 241.656054,103.186855 242.85397,103.186855 C244.187304,103.186855 245.275845,103.624355 246.119595,104.499355 C246.963345,105.363938 247.38522,106.567063 247.38522,108.10873 C247.38522,109.348313 247.19772,110.32748 246.82272,111.04623 C246.44772,111.754563 245.900845,112.306647 245.182095,112.70248 C244.473762,113.098313 243.69772,113.29623 242.85397,113.29623 C241.499804,113.29623 240.406054,112.863938 239.57272,111.999355 C238.739387,111.124355 238.32272,109.869147 238.32272,108.23373 Z M240.01022,108.23373 C240.01022,109.473313 240.275845,110.405605 240.807095,111.030605 C241.348762,111.645188 242.031054,111.95248 242.85397,111.95248 C243.66647,111.95248 244.343554,111.645188 244.88522,111.030605 C245.426887,110.405605 245.69772,109.457688 245.69772,108.186855 C245.69772,106.988938 245.426887,106.082688 244.88522,105.468105 C244.343554,104.853522 243.66647,104.54623 242.85397,104.54623 C242.031054,104.54623 241.348762,104.853522 240.807095,105.468105 C240.275845,106.072272 240.01022,106.994147 240.01022,108.23373 Z M252.885952,111.60873 L253.120327,113.061855 C252.661994,113.155605 252.250536,113.20248 251.885952,113.20248 C251.292202,113.20248 250.828661,113.10873 250.495327,112.92123 C250.172411,112.73373 249.943244,112.488938 249.807827,112.186855 C249.672411,111.874355 249.604702,111.228522 249.604702,110.249355 L249.604702,104.67123 L248.401577,104.67123 L248.401577,103.405605 L249.604702,103.405605 L249.604702,101.01498 L251.229702,100.030605 L251.229702,103.405605 L252.885952,103.405605 L252.885952,104.67123 L251.229702,104.67123 L251.229702,110.343105 C251.229702,110.801438 251.255744,111.098313 251.307827,111.23373 C251.370327,111.369147 251.464077,111.478522 251.589077,111.561855 C251.724494,111.634772 251.917202,111.67123 252.167202,111.67123 C252.344286,111.67123 252.583869,111.650397 252.885952,111.60873 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
index 7933f5805745e..a27ebecd9a9c7 100644
--- a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
+++ b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
@@ -67,22 +67,11 @@ If you use {es} {security-features}, you must have:
 Use the put enrich policy API
 to create a new <<enrich-policy,enrich policy>>.
 
-// tag::update-enrich-policy[]
 [WARNING]
 ====
-Once created, you can't update or change an enrich policy.
-Instead, you can:
-
-.   Create and <<execute-enrich-policy-api,execute>> a new enrich policy.
-
-.   Replace the previous enrich policy
-    with the new enrich policy
-    in any in-use enrich processors.
-
-.   Use the <<delete-enrich-policy-api, delete enrich policy>> API
-    to delete the previous enrich policy.
+include::../../enrich.asciidoc[tag=update-enrich-policy]
 ====
-// end::update-enrich-policy[]
+
 
 
 [[put-enrich-policy-api-path-params]]
diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc
index 5a8e0dab3a25a..98530032a9316 100644
--- a/docs/reference/ingest/enrich.asciidoc
+++ b/docs/reference/ingest/enrich.asciidoc
@@ -150,7 +150,11 @@ include::enrich.asciidoc[tag=enrich-policy-fields]
 You can use this definition to create the enrich policy with the
 <<put-enrich-policy-api,put enrich policy API>>.
 
-include::apis/enrich/put-enrich-policy.asciidoc[tag=update-enrich-policy]
+[WARNING]
+====
+Once created, you can't update or change an enrich policy.
+See <<update-enrich-policies>>.
+====
 
 [[execute-enrich-policy]]
 ==== Execute the enrich policy
@@ -214,7 +218,19 @@ using your ingest pipeline.
 [[update-enrich-policies]]
 ==== Update an enrich policy
 
-include::apis/enrich/put-enrich-policy.asciidoc[tag=update-enrich-policy]
+// tag::update-enrich-policy[]
+Once created, you can't update or change an enrich policy.
+Instead, you can:
+
+.   Create and <<execute-enrich-policy-api,execute>> a new enrich policy.
+
+.   Replace the previous enrich policy
+    with the new enrich policy
+    in any in-use enrich processors.
+
+.   Use the <<delete-enrich-policy-api, delete enrich policy>> API
+    to delete the previous enrich policy.
+// end::update-enrich-policy[]
 
 [role="xpack"]
 [testenv="basic"]
diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc
index ec3df1fa96347..b1b110e2b3889 100644
--- a/docs/reference/ingest/processors/inference.asciidoc
+++ b/docs/reference/ingest/processors/inference.asciidoc
@@ -14,7 +14,7 @@ ingested in the pipeline.
 | Name               | Required  | Default                        | Description
 | `model_id`         | yes       | -                              | (String) The ID of the model to load and infer against.
 | `target_field`     | no        | `ml.inference.<processor_tag>` | (String) Field added to incoming documents to contain results objects.
-| `field_mappings`   | yes       | -                              | (Object) Maps the document field names to the known field names of the model.
+| `field_map`        | yes       | -                              | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
 | `inference_config` | yes       | -                              | (Object) Contains the inference type and its options. There are two types: <<inference-processor-regression-opt,`regression`>> and <<inference-processor-classification-opt,`classification`>>.
 include::common-options.asciidoc[]
 |======
@@ -26,7 +26,7 @@ include::common-options.asciidoc[]
   "inference": {
     "model_id": "flight_delay_regression-1571767128603",
     "target_field": "FlightDelayMin_prediction_infer",
-    "field_mappings": {},
+    "field_map": {},
     "inference_config": { "regression": {} }
   }
 }
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index 1b5afad31f0c6..d26d897474deb 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -7,7 +7,7 @@ document:
 [float]
 === Core datatypes
 
-string::         <<text,`text`>> and <<keyword,`keyword`>>
+string::         <<text,`text`>>, <<keyword,`keyword`>> and <<wildcard,`wildcard`>>
 <<number>>::     `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`
 <<date>>::       `date`
 <<date_nanos>>:: `date_nanos`
@@ -131,3 +131,5 @@ include::types/token-count.asciidoc[]
 include::types/shape.asciidoc[]
 
 include::types/constant-keyword.asciidoc[]
+
+include::types/wildcard.asciidoc[]
diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc
index 440530b110247..6d1dbe49b131d 100644
--- a/docs/reference/mapping/types/histogram.asciidoc
+++ b/docs/reference/mapping/types/histogram.asciidoc
@@ -69,7 +69,6 @@ The following <<indices-create-index, create index>> API request creates a new i
 * `my_histogram`, a `histogram` field used to store percentile data
 * `my_text`, a `keyword` field used to store a title for the histogram
 
-[ INSERT CREATE INDEX SNIPPET ]
 [source,console]
 --------------------------------------------------
 PUT my_index
diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc
index 75cb76a4faff4..da4cb14ba9904 100644
--- a/docs/reference/mapping/types/search-as-you-type.asciidoc
+++ b/docs/reference/mapping/types/search-as-you-type.asciidoc
@@ -162,15 +162,18 @@ GET my_index/_search
 The following parameters are accepted in a mapping for the `search_as_you_type`
 field and are specific to this field type
 
-[horizontal]
-
 `max_shingle_size`::
++
+--
+(Optional, integer)
+Largest shingle size to create. Valid values are `2` (inclusive) to `4`
+(inclusive). Defaults to `3`.
 
-    The largest shingle size to index the input with and create subfields for,
-    creating one subfield for each shingle size between 2 and
-    `max_shingle_size`. Accepts integer values between 2 and 4 inclusive. This
-    option defaults to 3.
+A subfield is created for each integer between `2` and this value. For example,
+a value of `3` creates two subfields: `my_field._2gram` and `my_field._3gram`
 
+More subfields enables more specific queries but increases index size.
+--
 
 [[general-params]]
 ==== Parameters of the field type as a text field
diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
new file mode 100644
index 0000000000000..51d10ff53ca92
--- /dev/null
+++ b/docs/reference/mapping/types/wildcard.asciidoc
@@ -0,0 +1,53 @@
+[role="xpack"]
+[testenv="basic"]
+[[wildcard]]
+=== Wildcard datatype
+++++
+<titleabbrev>Wildcard</titleabbrev>
+++++
+
+A `wildcard` field stores values optimised for wildcard grep-like queries.
+Wildcard queries are possible on other field types but suffer from constraints:
+* `text` fields limit matching of any wildcard expressions to individual tokens rather than the original whole value held in a field
+* `keyword` fields are untokenized but slow at performing wildcard queries (especially patterns with leading wildcards).
+
+Internally the `wildcard` field indexes the whole field value using ngrams and stores the full string.
+The index is used as a rough filter to cut down the number of values that are then checked by retrieving and checking the full values.
+This field is especially well suited to run grep-like queries on log lines. Storage costs are typically lower than those of `keyword`
+fields but search speeds for exact matches on full terms are slower.
+
+You index and search a wildcard field as follows
+
+[source,console]
+--------------------------------------------------
+PUT my_index
+{
+  "mappings": {
+    "properties": {
+      "my_wildcard": {
+        "type": "wildcard"
+      }
+    }
+  }
+}
+
+PUT my_index/_doc/1
+{
+  "my_wildcard" : "This string can be quite lengthy"
+}
+
+POST my_index/_doc/_search
+{
+  "query": {
+      "wildcard" : "*quite*lengthy"
+  }
+}
+
+
+--------------------------------------------------
+
+
+==== Limitations
+
+* `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries.
+
diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc
index 6a5b319616096..9e52ddb770563 100644
--- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc
+++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc
@@ -94,3 +94,10 @@ breaking change was made necessary by
 https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[AWS's
 announcement] that the path-style access pattern is deprecated and will be
 unsupported on buckets created after September 30th 2020.
+
+[float]
+==== Restore requests no longer accept settings
+
+In earlier versions, you could pass both `settings` and `index_settings` in the
+body of a restore snapshot request, but the `settings` value was ignored. The
+restore snapshot API now rejects requests that include a `settings` value.
diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
index d9087a6dac51b..6a011dcb04f48 100644
--- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
@@ -124,9 +124,9 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=eta]
 (Optional, double) 
 include::{docdir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction]
 
-`analysis`.`classification`.`maximum_number_trees`::::
+`analysis`.`classification`.`max_trees`::::
 (Optional, integer) 
-include::{docdir}/ml/ml-shared.asciidoc[tag=maximum-number-trees]
+include::{docdir}/ml/ml-shared.asciidoc[tag=max-trees]
 
 `analysis`.`classification`.`gamma`::::
 (Optional, double) 
@@ -136,6 +136,10 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=gamma]
 (Optional, double) 
 include::{docdir}/ml/ml-shared.asciidoc[tag=lambda]
 
+`analysis`.`classification`.`class_assignment_objective`::::
+(Optional, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=class-assignment-objective]
+
 `analysis`.`classification`.`num_top_classes`::::
 (Optional, integer)
 include::{docdir}/ml/ml-shared.asciidoc[tag=num-top-classes]
@@ -218,9 +222,9 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=eta]
 (Optional, double)
 include::{docdir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction]
 
-`analysis`.`regression`.`maximum_number_trees`::::
+`analysis`.`regression`.`max_trees`::::
 (Optional, integer) 
-include::{docdir}/ml/ml-shared.asciidoc[tag=maximum-number-trees]
+include::{docdir}/ml/ml-shared.asciidoc[tag=max-trees]
 
 `analysis`.`regression`.`gamma`::::
 (Optional, double) 
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index ce0a06174a24a..16c9f486d8d63 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -339,6 +339,14 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=mode]
 include::{docdir}/ml/ml-shared.asciidoc[tag=time-span]
 end::chunking-config[]
 
+tag::class-assignment-objective[]
+Defines the objective to optimize when assigning class labels. Available
+objectives are `maximize_accuracy` and `maximize_minimum_recall`. When maximizing
+accuracy class labels are chosen to maximize the number of correct predictions.
+When maximizing minimum recall labels are chosen to maximize the minimum recall
+for any class. Defaults to maximize_minimum_recall.
+end::class-assignment-objective[]
+
 tag::custom-rules[]
 An array of custom rule objects, which enable you to customize the way detectors
 operate. For example, a rule may dictate to the detector conditions under which
@@ -946,10 +954,10 @@ remain started until it is explicitly stopped. By default this setting is not
 set.
 end::max-empty-searches[]
 
-tag::maximum-number-trees[]
+tag::max-trees[]
 Advanced configuration option. Defines the maximum number of trees the forest is
 allowed to contain. The maximum value is 2000.
-end::maximum-number-trees[]
+end::max-trees[]
 
 tag::memory-estimation[]
 An object containing the memory estimates. The object has the 
@@ -1317,6 +1325,11 @@ The configuration of how to source the analysis data. It requires an
   (Required, string or array) Index or indices on which to perform the 
   analysis. It can be a single index or index pattern as well as an array of 
   indices or patterns.
++
+--
+WARNING: If your source indices contain documents with the same IDs, only the 
+document that is indexed last appears in the destination index.
+--
   
 `query`:::
   (Optional, object) The {es} query domain-specific language 
@@ -1506,6 +1519,17 @@ The estimated number of operations to use the trained model.
 `license_level`:::
 (string)
 The license level of the trained model.
+
+`default_field_map` :::
+(object)
+A string to string object that contains the default field map to use
+when inferring against the model. For example, data frame analytics
+may train the model on a specific multi-field `foo.keyword`.
+The analytics job would then supply a default field map entry for
+`"foo" : "foo.keyword"`.
+
+Any field map described in the inference configuration takes precedence.
+
 end::trained-model-configs[]
 
 tag::training-percent[]
diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc
index 417acff5a5254..69f78f1364c52 100644
--- a/docs/reference/modules/cross-cluster-search.asciidoc
+++ b/docs/reference/modules/cross-cluster-search.asciidoc
@@ -253,30 +253,33 @@ PUT _cluster/settings
 If `cluster_two` is disconnected or unavailable during a {ccs}, {es} won't
 include matching documents from that cluster in the final results.
 
-[discrete]
-[[ccs-works]]
-== How {ccs} works
-
-include::./remote-clusters.asciidoc[tag=how-remote-clusters-work]
-
 [discrete]
 [[ccs-gateway-seed-nodes]]
-=== Selecting gateway and seed nodes
+== Selecting gateway and seed nodes in sniff mode
 
-Gateway and seed nodes need to be accessible from the local cluster via your
-network.
+For remote clusters using the <<sniff-mode,sniff connection>> mode, gateway and
+seed nodes need to be accessible from the local cluster via your network.
 
-By default, any master-ineligible node can act as a gateway node. If wanted,
-you can define the gateway nodes for a cluster by setting
-`cluster.remote.node.attr.gateway` to `true`.
+By default, any non-<<master-node,master-eligible>> node can act as a
+gateway node. If wanted, you can define the gateway nodes for a cluster by
+setting `cluster.remote.node.attr.gateway` to `true`.
 
 For {ccs}, we recommend you use gateway nodes that are capable of serving as
 <<coordinating-node,coordinating nodes>> for search requests. If
 wanted, the seed nodes for a cluster can be a subset of these gateway nodes.
 
+[discrete]
+[[ccs-proxy-mode]]
+== {ccs} in proxy mode
+
+<<proxy-mode,Proxy mode>> remote cluster connections support {ccs}. All remote
+connections connect to the configured `proxy_address`. Any desired connection
+routing to gateway or <<coordinating-node,coordinating nodes>> must
+be implemented by the intermediate proxy at this configured address.
+
 [discrete]
 [[ccs-network-delays]]
-=== How {ccs} handles network delays
+== How {ccs} handles network delays
 
 Because {ccs} involves sending requests to remote clusters, any network delays
 can impact search speed. To avoid slow searches, {ccs} offers two options for
@@ -300,9 +303,9 @@ low latency.
 +
 See <<ccs-unmin-roundtrips>> to learn how this option works.
 
-[float]
+[discrete]
 [[ccs-min-roundtrips]]
-==== Minimize network roundtrips
+=== Minimize network roundtrips
 
 Here's how {ccs} works when you minimize network roundtrips.
 
@@ -326,9 +329,9 @@ final results in the {ccs} response.
 +
 image:images/ccs/ccs-min-roundtrip-client-response.svg[]
 
-[float]
+[discrete]
 [[ccs-unmin-roundtrips]]
-==== Don't minimize network roundtrips
+=== Don't minimize network roundtrips
 
 Here's how {ccs} works when you don't minimize network roundtrips.
 
diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc
index 0d0b24f31e25d..07b9fd23f96dc 100644
--- a/docs/reference/modules/remote-clusters.asciidoc
+++ b/docs/reference/modules/remote-clusters.asciidoc
@@ -13,23 +13,31 @@ connections to a remote cluster. This functionality is used in
 <<modules-cross-cluster-search,{ccs}>>.
 endif::[]
 
-// tag::how-remote-clusters-work[]
 Remote cluster connections work by configuring a remote cluster and connecting
-only to a limited number of nodes in that remote cluster. Each remote cluster
-is referenced by a name and a list of seed nodes. When a remote cluster is
-registered, its cluster state is retrieved from one of the seed nodes and up
-to three _gateway nodes_ are selected to be connected to as part of remote
-cluster requests.
-// end::how-remote-clusters-work[]
+to a limited number of nodes in that remote cluster. There are two modes for
+remote cluster connections: <<sniff-mode,sniff mode>> and
+<<proxy-mode,proxy mode>>.
 
 All the communication required between different clusters
 goes through the <<modules-transport,transport layer>>. Remote cluster
 connections consist of uni-directional connections from the coordinating
-node to the selected remote _gateway nodes_ only.
+node to the remote remote connections.
+
+[float]
+[[sniff-mode]]
+=== Sniff mode
+
+In sniff mode, a cluster is created using a name and a list of seed nodes. When
+a remote cluster is registered, its cluster state is retrieved from one of the
+seed nodes and up to three _gateway nodes_ are selected as part of remote
+cluster requests. This mode requires that the gateway node's publish addresses
+are accessible by the local cluster.
+
+Sniff mode is the default connection mode.
 
 [float]
 [[gateway-nodes-selection]]
-=== Gateway nodes selection
+==== Gateway nodes selection
 
 The _gateway nodes_ selection depends on the following criteria:
 
@@ -62,9 +70,24 @@ communicate with 6.7. The matrix below summarizes compatibility as described abo
 (see <<remote-cluster-settings>>), though such tagged nodes still have
 to satisfy the two above requirements.
 
+[float]
+[[proxy-mode]]
+=== Proxy mode
+
+In proxy mode, a cluster is created using a name and a single proxy address. When
+a remote cluster is registered, a configurable number of socket connections are
+opened to the proxy address. The proxy is required to route those connections to
+the remote cluster. Proxy mode does not require remote cluster nodes to have
+accessible publish addresses.
+
+The proxy mode is not the default connection mode and must be configured. Similar
+to the sniff <<gateway-nodes-selection,gateway nodes>>, the remote
+connections are subject to the same version compatibility rules as
+<<rolling-upgrades>>.
+
 [float]
 [[configuring-remote-clusters]]
-=== Configuring remote clusters
+==== Configuring remote clusters
 
 You can configure remote clusters globally by using
 <<cluster-update-settings,cluster settings>>, which you can update dynamically.
@@ -83,23 +106,32 @@ cluster:
         cluster_one: <1>
             seeds: 127.0.0.1:9300 <2>
             transport.ping_schedule: 30s <3>
-        cluster_two: 
-            seeds: 127.0.0.1:9301
-            transport.compress: true <4>
-            skip_unavailable: true <5>
+        cluster_two: <1>
+            mode: sniff <4>
+            seeds: 127.0.0.1:9301 <2>
+            transport.compress: true <5>
+            skip_unavailable: true <6>
+        cluster_three: <1>
+            mode: proxy <4>
+            proxy_address: 127.0.0.1:9302 <7>
 
 --------------------------------
-<1> `cluster_one` and `cluster_two` are arbitrary _cluster aliases_ representing
-the connection to each cluster. These names are subsequently used to distinguish
-between local and remote indices.
+<1> `cluster_one`, `cluster_two`, and `cluster_three` are arbitrary _cluster aliases_
+representing the connection to each cluster. These names are subsequently used to
+distinguish between local and remote indices.
 <2> The hostname and <<modules-transport,transport>> port (default: 9300) of a
 seed node in the remote cluster.
 <3> A keep-alive ping is configured for `cluster_one`.
-<4> Compression is explicitly enabled for requests to `cluster_two`.
-<5> Disconnected remote clusters are optional for `cluster_two`.
+<4> The configured connection mode. By default, this is <<sniff-mode,`sniff`>>, so
+the mode is implicit for `cluster_one`. However, it can be explicitly configured
+as demonstrated by `cluster_two` and must be explicitly configured for
+<<proxy-mode,proxy mode>> as demonstrated by `cluster_three`.
+<5> Compression is explicitly enabled for requests to `cluster_two`.
+<6> Disconnected remote clusters are optional for `cluster_two`.
+<7> The address for the proxy endpoint used to connect to `cluster_three`.
 
 For more information about the optional transport settings, see
-<<modules-transport>>. 
+<<modules-transport>>.
 
 
 If you use <<cluster-update-settings,cluster settings>>, the remote clusters
@@ -119,6 +151,7 @@ PUT _cluster/settings
           "transport.ping_schedule": "30s"
         },
         "cluster_two": {
+          "mode": "sniff",
           "seeds": [
             "127.0.0.1:9301"
           ],
@@ -126,9 +159,8 @@ PUT _cluster/settings
           "skip_unavailable": true
         },
         "cluster_three": {
-          "seeds": [
-            "127.0.0.1:9302"
-          ]
+          "mode": "proxy",
+          "proxy_address": "127.0.0.1:9302"
         }
       }
     }
@@ -139,7 +171,8 @@ PUT _cluster/settings
 // TEST[s/127.0.0.1:9300/\${transport_host}/]
 
 You can dynamically update the compression and ping schedule settings. However,
-you must re-include seeds in the settings update request. For example:
+you must re-include seeds or `proxy_address` in the settings update request.
+For example:
 
 [source,console]
 --------------------------------
@@ -155,10 +188,16 @@ PUT _cluster/settings
           "transport.ping_schedule": "60s"
         },
         "cluster_two": {
+          "mode": "sniff",
           "seeds": [
             "127.0.0.1:9301"
           ],
           "transport.compress": false
+        },
+        "cluster_three": {
+          "mode": "proxy",
+          "proxy_address": "127.0.0.1:9302",
+          "transport.compress": true
         }
       }
     }
@@ -171,7 +210,7 @@ NOTE: When the compression or ping schedule settings change, all the existing
 node connections must close and re-open, which can cause in-flight requests to
 fail.
 
-A remote cluster can be deleted from the cluster settings by setting its seeds and optional settings to `null` :
+A remote cluster can be deleted from the cluster settings by setting its settings to `null` :
 
 [source,console]
 --------------------------------
@@ -181,6 +220,7 @@ PUT _cluster/settings
     "cluster": {
       "remote": {
         "cluster_two": { <1>
+          "mode": null,
           "seeds": null,
           "skip_unavailable": null,
           "transport": {
@@ -199,25 +239,21 @@ PUT _cluster/settings
 
 [float]
 [[remote-cluster-settings]]
-=== Remote cluster settings
+=== Remote cluster settings for all modes
 
-`cluster.remote.connections_per_cluster`::
+These settings apply to both <<sniff-mode,sniff mode>> and
+<<proxy-mode,proxy mode>>. <<remote-cluster-sniff-settings,Sniff mode settings>>
+and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
 
-  The number of gateway nodes to connect to per remote cluster. The default is
-  `3`.
+`cluster.remote.<cluster_alias>.mode`::
+  The mode used for a remote cluster connection. The only supported modes are
+  `sniff` and `proxy`.
 
 `cluster.remote.initial_connect_timeout`::
 
   The time to wait for remote connections to be established when the node
   starts. The default is `30s`.
 
-`cluster.remote.node.attr`::
-
-  A node attribute to filter out nodes that are eligible as a gateway node in
-  the remote cluster. For instance a node can have a node attribute
-  `node.attr.gateway: true` such that only nodes with this attribute will be
-  connected to if `cluster.remote.node.attr` is set to `gateway`.
-
 `cluster.remote.connect`::
 
   By default, any node in the cluster can act as a cross-cluster client and
@@ -226,7 +262,7 @@ PUT _cluster/settings
   remote clusters. Remote cluster requests must be sent to a node that is
   allowed to act as a cross-cluster client.
 
-`cluster.remote.${cluster_alias}.skip_unavailable`::
+`cluster.remote.<cluster_alias>.skip_unavailable`::
 
   Per cluster boolean setting that allows to skip specific clusters when no
   nodes belonging to them are available and they are the targetof a remote
@@ -234,7 +270,7 @@ PUT _cluster/settings
   by default, but they can selectively be made optional by setting this setting
   to `true`.
 
-`cluster.remote.${cluster_alias}.transport.ping_schedule`::
+`cluster.remote.<cluster_alias>.transport.ping_schedule`::
 
   Sets the time interval between regular application-level ping messages that
   are sent to ensure that transport connections to nodes belonging to remote
@@ -243,7 +279,7 @@ PUT _cluster/settings
   are sent according to the global `transport.ping_schedule` setting, which
   defaults to `-1` meaning that pings are not sent.
 
-`cluster.remote.${cluster_alias}.transport.compress`::
+`cluster.remote.<cluster_alias>.transport.compress`::
 
   Per cluster boolean setting that enables you to configure compression for
   requests to a specific remote cluster. This setting impacts only requests
@@ -251,6 +287,48 @@ PUT _cluster/settings
   Elasticsearch compresses the response. If unset, the global
   `transport.compress` is used as the fallback setting.
 
+[float]
+[[remote-cluster-sniff-settings]]
+=== Remote cluster settings for sniff mode
+
+`cluster.remote.<cluster_alias>.seeds`::
+
+  The list of seed nodes used to sniff the remote cluster state.
+
+`cluster.remote.<cluster_alias>.node_connections`::
+
+  The number of gateway nodes to connect to for this remote cluster. The default
+  is `3`.
+
+`cluster.remote.node.attr`::
+
+  A node attribute to filter out nodes that are eligible as a gateway node in
+  the remote cluster. For instance a node can have a node attribute
+  `node.attr.gateway: true` such that only nodes with this attribute will be
+  connected to if `cluster.remote.node.attr` is set to `gateway`.
+
+[float]
+[[remote-cluster-proxy-settings]]
+=== Remote cluster settings for proxy mode
+
+`cluster.remote.<cluster_alias>.proxy_address`::
+
+  The address used for all remote connections.
+
+`cluster.remote.<cluster_alias>.proxy_socket_connections`::
+
+  The number of socket connections to open per remote cluster. The default is
+  `18`.
+
+[role="xpack"]
+`cluster.remote.<cluster_alias>.server_name`::
+
+  An optional hostname string which is sent in the `server_name` field of
+  the TLS Server Name Indication extension if
+  <<configuring-tls,TLS is enabled>>. The TLS transport will fail to open
+  remote connections if this field is not a valid hostname as defined by the
+  TLS SNI specification.
+
 [float]
 [[retrieve-remote-clusters-info]]
 === Retrieving remote clusters info
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 9e6fe18d754d0..9ab175ee4d8e4 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -370,3 +370,13 @@ See <<slm-api-start>>.
 === Stop {slm} API
 
 See <<slm-api-stop>>.
+
+[role="exclude",id="ccs-works"]
+=== How {ccs} works
+
+See <<ccs-gateway-seed-nodes>> and <<ccs-min-roundtrips>>.
+
+[role="exclude",id="async-search"]
+=== Asynchronous search
+
+coming::[7.x]
diff --git a/docs/reference/search/request/stored-fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc
index c1f0feaed96d3..bca21a9da4fc6 100644
--- a/docs/reference/search/request/stored-fields.asciidoc
+++ b/docs/reference/search/request/stored-fields.asciidoc
@@ -40,12 +40,7 @@ If the requested fields are not stored (`store` mapping set to `false`), they wi
 
 Stored field values fetched from the document itself are always returned as an array. On the contrary, metadata fields like `_routing` are never returned as an array.
 
-Also only leaf fields can be returned via the `field` option. So object fields can't be returned and such requests
-will fail.
-
-Script fields can also be automatically detected and used as fields, so
-things like `_source.obj1.field1` can be used, though not recommended, as
-`obj1.field1` will work as well.
+Also only leaf fields can be returned via the `stored_fields` option. If an object field is specified, it will be ignored.
 
 NOTE: On its own, `stored_fields` cannot be used to load fields in nested
 objects -- if a field contains a nested object in its path, then no data will
diff --git a/docs/reference/sql/language/data-types.asciidoc b/docs/reference/sql/language/data-types.asciidoc
index 811bb1ac6a47f..2c799dc7276eb 100644
--- a/docs/reference/sql/language/data-types.asciidoc
+++ b/docs/reference/sql/language/data-types.asciidoc
@@ -13,21 +13,22 @@ s|SQL precision
 
 4+h| Core types
 
-| <<null-value, `null`>>      | null          | NULL        | 0
-| <<boolean, `boolean`>>      | boolean       | BOOLEAN     | 1
-| <<number, `byte`>>          | byte          | TINYINT     | 3
-| <<number, `short`>>         | short         | SMALLINT    | 5
-| <<number, `integer`>>       | integer       | INTEGER     | 10
-| <<number, `long`>>          | long          | BIGINT      | 19
-| <<number, `double`>>        | double        | DOUBLE      | 15
-| <<number, `float`>>         | float         | REAL        | 7
-| <<number, `half_float`>>    | half_float    | FLOAT       | 3
-| <<number, `scaled_float`>>  | scaled_float  | DOUBLE      | 15
-| <<keyword, `keyword`>>      | keyword       | VARCHAR     | 32,766
-| <<text, `text`>>            | text          | VARCHAR     | 2,147,483,647
-| <<binary, `binary`>>        | binary        | VARBINARY   | 2,147,483,647
-| <<date, `date`>>            | datetime      | TIMESTAMP   | 29
-| <<ip, `ip`>>                | ip            | VARCHAR     | 39
+| <<null-value, `null`>>                   | null            | NULL        | 0
+| <<boolean, `boolean`>>                   | boolean         | BOOLEAN     | 1
+| <<number, `byte`>>                       | byte            | TINYINT     | 3
+| <<number, `short`>>                      | short           | SMALLINT    | 5
+| <<number, `integer`>>                    | integer         | INTEGER     | 10
+| <<number, `long`>>                       | long            | BIGINT      | 19
+| <<number, `double`>>                     | double          | DOUBLE      | 15
+| <<number, `float`>>                      | float           | REAL        | 7
+| <<number, `half_float`>>                 | half_float      | FLOAT       | 3
+| <<number, `scaled_float`>>               | scaled_float    | DOUBLE      | 15
+| <<keyword, `keyword`>>                   | keyword         | VARCHAR     | 32,766
+| <<constant-keyword, `constant_keyword`>> | constant_keyword| VARCHAR     | 32,766
+| <<text, `text`>>                         | text            | VARCHAR     | 2,147,483,647
+| <<binary, `binary`>>                     | binary          | VARBINARY   | 2,147,483,647
+| <<date, `date`>>                         | datetime        | TIMESTAMP   | 29
+| <<ip, `ip`>>                             | ip              | VARCHAR     | 39
 
 4+h| Complex types
 
diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc
index e23a6fb2fd54b..1175fba036bb4 100644
--- a/docs/reference/transform/apis/get-transform-stats.asciidoc
+++ b/docs/reference/transform/apis/get-transform-stats.asciidoc
@@ -104,6 +104,10 @@ upper bound of data that is included in the checkpoint.
 `checkpointing`.`next`.`timestamp_millis`::::
 (date) The timestamp of the checkpoint, which indicates when the checkpoint was
 created.
+`checkpointing`.`operations_behind`:::
+(integer) The number of operations that have occurred on the source index but
+have not been applied to the destination index yet. A high number can indicate
+that the {transform} is failing to keep up. 
 
 `id`::
 (string)
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
index e0ea0d511e277..8303cef383f44 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
@@ -47,6 +47,31 @@ public abstract class AbstractObjectParser<Value, Context>
     public abstract <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Context, T> parser, ParseField parseField,
             ValueType type);
 
+    /**
+     * Declares a single named object.
+     *
+     * <pre>
+     * <code>
+     * {
+     *   "object_name": {
+     *     "instance_name": { "field1": "value1", ... }
+     *     }
+     *   }
+     * }
+     * </code>
+     * </pre>
+     *
+     * @param consumer
+     *            sets the value once it has been parsed
+     * @param namedObjectParser
+     *            parses the named object
+     * @param parseField
+     *            the field to parse
+     */
+    public abstract <T> void declareNamedObject(BiConsumer<Value, T> consumer, NamedObjectParser<T, Context> namedObjectParser,
+                                                 ParseField parseField);
+
+
     /**
      * Declares named objects in the style of aggregations. These are named
      * inside and object like this:
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
index 099701e688136..52bfcc651609e 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
@@ -206,16 +206,15 @@ public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Contex
             throw new IllegalArgumentException("[type] is required");
         }
 
-        if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
+        if (isConstructorArg(consumer)) {
             /*
-             * Constructor arguments are detected by these "marker" consumers. It keeps the API looking clean even if it is a bit sleezy. We
-             * then build a new consumer directly against the object parser that triggers the "constructor arg just arrived behavior" of the
-             * parser. Conveniently, we can close over the position of the constructor in the argument list so we don't need to do any fancy
+             * Build a new consumer directly against the object parser that
+             * triggers the "constructor arg just arrived behavior" of the
+             * parser. Conveniently, we can close over the position of the
+             * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = constructorArgInfos.size();
-            boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
-            constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
+            int position = addConstructorArg(consumer, parseField);
             objectParser.declareField((target, v) -> target.constructorArg(position, v), parser, parseField, type);
         } else {
             numberOfFields += 1;
@@ -223,9 +222,39 @@ public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Contex
         }
     }
 
+    @Override
+    public <T> void declareNamedObject(BiConsumer<Value, T> consumer, NamedObjectParser<T, Context> namedObjectParser,
+                                                ParseField parseField) {
+        if (consumer == null) {
+            throw new IllegalArgumentException("[consumer] is required");
+        }
+        if (namedObjectParser == null) {
+            throw new IllegalArgumentException("[parser] is required");
+        }
+        if (parseField == null) {
+            throw new IllegalArgumentException("[parseField] is required");
+        }
+
+        if (isConstructorArg(consumer)) {
+            /*
+             * Build a new consumer directly against the object parser that
+             * triggers the "constructor arg just arrived behavior" of the
+             * parser. Conveniently, we can close over the position of the
+             * constructor in the argument list so we don't need to do any fancy
+             * or expensive lookups whenever the constructor args come in.
+             */
+            int position = addConstructorArg(consumer, parseField);
+            objectParser.declareNamedObject((target, v) -> target.constructorArg(position, v), namedObjectParser, parseField);
+        } else {
+            numberOfFields += 1;
+            objectParser.declareNamedObject(queueingConsumer(consumer, parseField), namedObjectParser, parseField);
+        }
+    }
+
     @Override
     public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedObjectParser<T, Context> namedObjectParser,
             ParseField parseField) {
+
         if (consumer == null) {
             throw new IllegalArgumentException("[consumer] is required");
         }
@@ -236,19 +265,15 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
             throw new IllegalArgumentException("[parseField] is required");
         }
 
-        if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
+        if (isConstructorArg(consumer)) {
             /*
-             * Constructor arguments are detected by this "marker" consumer. It
-             * keeps the API looking clean even if it is a bit sleezy. We then
-             * build a new consumer directly against the object parser that
+             * Build a new consumer directly against the object parser that
              * triggers the "constructor arg just arrived behavior" of the
              * parser. Conveniently, we can close over the position of the
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = constructorArgInfos.size();
-            boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
-            constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
+            int position = addConstructorArg(consumer, parseField);
             objectParser.declareNamedObjects((target, v) -> target.constructorArg(position, v), namedObjectParser, parseField);
         } else {
             numberOfFields += 1;
@@ -272,19 +297,15 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
             throw new IllegalArgumentException("[parseField] is required");
         }
 
-        if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
+        if (isConstructorArg(consumer)) {
             /*
-             * Constructor arguments are detected by this "marker" consumer. It
-             * keeps the API looking clean even if it is a bit sleezy. We then
-             * build a new consumer directly against the object parser that
+             * Build a new consumer directly against the object parser that
              * triggers the "constructor arg just arrived behavior" of the
              * parser. Conveniently, we can close over the position of the
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = constructorArgInfos.size();
-            boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
-            constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
+            int position = addConstructorArg(consumer, parseField);
             objectParser.declareNamedObjects((target, v) -> target.constructorArg(position, v), namedObjectParser,
                     wrapOrderedModeCallBack(orderedModeCallback), parseField);
         } else {
@@ -294,6 +315,27 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
         }
     }
 
+    /**
+     * Constructor arguments are detected by this "marker" consumer. It
+     * keeps the API looking clean even if it is a bit sleezy.
+     */
+    private boolean isConstructorArg(BiConsumer<?, ?> consumer) {
+        return consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER;
+    }
+
+    /**
+     * Add a constructor argument
+     * @param consumer Either {@link #REQUIRED_CONSTRUCTOR_ARG_MARKER} or {@link #REQUIRED_CONSTRUCTOR_ARG_MARKER}
+     * @param parseField Parse field
+     * @return The argument position
+     */
+    private int addConstructorArg(BiConsumer<?, ?> consumer, ParseField parseField) {
+        int position = constructorArgInfos.size();
+        boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
+        constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
+        return position;
+    }
+    
     @Override
     public String getName() {
         return objectParser.getName();
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index 7526fc7bd6922..b11f6afd4bb26 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -394,6 +394,32 @@ public <T> void declareObjectOrDefault(BiConsumer<Value, T> consumer, BiFunction
         }, field, ValueType.OBJECT_OR_BOOLEAN);
     }
 
+    @Override
+    public <T> void declareNamedObject(BiConsumer<Value, T> consumer, NamedObjectParser<T, Context> namedObjectParser,
+                                       ParseField field) {
+
+        BiFunction<XContentParser, Context, T> objectParser = (XContentParser p, Context c) -> {
+            try {
+                XContentParser.Token token = p.nextToken();
+                assert token == XContentParser.Token.FIELD_NAME;
+                String name = p.currentName();
+                try {
+                    T namedObject = namedObjectParser.parse(p, c, name);
+                    // consume the end object token
+                    token = p.nextToken();
+                    assert token == XContentParser.Token.END_OBJECT;
+                    return namedObject;
+                } catch (Exception e) {
+                    throw new XContentParseException(p.getTokenLocation(), "[" + field + "] failed to parse field [" + name + "]", e);
+                }
+            } catch (IOException e) {
+                throw new XContentParseException(p.getTokenLocation(), "[" + field + "] error while parsing named object", e);
+            }
+        };
+
+        declareField((XContentParser p, Value v, Context c) -> consumer.accept(v, objectParser.apply(p, c)), field, ValueType.OBJECT);
+    }
+
     @Override
     public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedObjectParser<T, Context> namedObjectParser,
             Consumer<Value> orderedModeCallback, ParseField field) {
@@ -403,7 +429,7 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
                 throw new XContentParseException(p.getTokenLocation(), "[" + field + "] can be a single object with any number of "
                         + "fields or an array where each entry is an object with a single field");
             }
-            // This messy exception nesting has the nice side effect of telling the use which field failed to parse
+            // This messy exception nesting has the nice side effect of telling the user which field failed to parse
             try {
                 String name = p.currentName();
                 try {
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index 3e8d5dee8614f..c99d1b10d6a4d 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -500,55 +500,70 @@ public void setString_or_null(String string_or_null) {
     }
 
     public void testParseNamedObject() throws IOException {
-        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": { \"a\": {} }}");
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
+                "{\"named\": { \"a\": {\"foo\" : 11} }, \"bar\": \"baz\"}");
         NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
+        assertEquals("a", h.named.name);
+        assertEquals(11, h.named.foo);
+        assertEquals("baz", h.bar);
+    }
+
+    public void testParseNamedObjectUnexpectedArray() throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ \"a\": {\"foo\" : 11} }]");
+        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
+        assertThat(e.getMessage(), containsString("[named_object_holder] named doesn't support values of type: START_ARRAY"));
+    }
+
+    public void testParseNamedObjects() throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": { \"a\": {} }}");
+        NamedObjectsHolder h = NamedObjectsHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertFalse(h.namedSuppliedInOrder);
     }
 
-    public void testParseNamedObjectInOrder() throws IOException {
+    public void testParseNamedObjectsInOrder() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {\"a\": {}} ] }");
-        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
+        NamedObjectsHolder h = NamedObjectsHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertTrue(h.namedSuppliedInOrder);
     }
 
-    public void testParseNamedObjectTwoFieldsInArray() throws IOException {
+    public void testParseNamedObjectsTwoFieldsInArray() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {\"a\": {}, \"b\": {}}]}");
-        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
-        assertThat(e.getMessage(), containsString("[named_object_holder] failed to parse field [named]"));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectsHolder.PARSER.apply(parser, null));
+        assertThat(e.getMessage(), containsString("[named_objects_holder] failed to parse field [named]"));
         assertThat(e.getCause().getMessage(),
                 containsString("[named] can be a single object with any number of fields " +
                     "or an array where each entry is an object with a single field"));
     }
 
-    public void testParseNamedObjectNoFieldsInArray() throws IOException {
+    public void testParseNamedObjectsNoFieldsInArray() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {} ]}");
-        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
-        assertThat(e.getMessage(), containsString("[named_object_holder] failed to parse field [named]"));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectsHolder.PARSER.apply(parser, null));
+        assertThat(e.getMessage(), containsString("[named_objects_holder] failed to parse field [named]"));
         assertThat(e.getCause().getMessage(),
                 containsString("[named] can be a single object with any number of fields " +
                     "or an array where each entry is an object with a single field"));
     }
 
-    public void testParseNamedObjectJunkInArray() throws IOException {
+    public void testParseNamedObjectsJunkInArray() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ \"junk\" ] }");
-        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
-        assertThat(e.getMessage(), containsString("[named_object_holder] failed to parse field [named]"));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> NamedObjectsHolder.PARSER.apply(parser, null));
+        assertThat(e.getMessage(), containsString("[named_objects_holder] failed to parse field [named]"));
         assertThat(e.getCause().getMessage(),
                 containsString("[named] can be a single object with any number of fields " +
                     "or an array where each entry is an object with a single field"));
     }
 
-    public void testParseNamedObjectInOrderNotSupported() throws IOException {
+    public void testParseNamedObjectsInOrderNotSupported() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"named\": [ {\"a\": {}} ] }");
 
         // Create our own parser for this test so we can disable support for the "ordered" mode specified by the array above
-        ObjectParser<NamedObjectHolder, Void> objectParser = new ObjectParser<>("named_object_holder",
-                NamedObjectHolder::new);
-        objectParser.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
+        ObjectParser<NamedObjectsHolder, Void> objectParser = new ObjectParser<>("named_object_holder",
+                NamedObjectsHolder::new);
+        objectParser.declareNamedObjects(NamedObjectsHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
 
         // Now firing the xml through it fails
         XContentParseException e = expectThrows(XContentParseException.class, () -> objectParser.apply(parser, null));
@@ -713,7 +728,7 @@ public void testNoopDeclareField() throws IOException {
         assertEquals("parser for [noop] did not end on END_ARRAY", e.getMessage());
     }
 
-    public void testNoopDeclareObjectArray() throws IOException {
+    public void testNoopDeclareObjectArray() {
         ObjectParser<AtomicReference<String>, Void> parser = new ObjectParser<>("noopy", AtomicReference::new);
         parser.declareString(AtomicReference::set, new ParseField("body"));
         parser.declareObjectArray((a,b) -> {}, (p, c) -> null, new ParseField("noop"));
@@ -728,11 +743,33 @@ public void testNoopDeclareObjectArray() throws IOException {
         assertEquals("expected value but got [FIELD_NAME]", sneakyError.getCause().getMessage());
     }
 
+    // singular
     static class NamedObjectHolder {
         public static final ObjectParser<NamedObjectHolder, Void> PARSER = new ObjectParser<>("named_object_holder",
                 NamedObjectHolder::new);
         static {
-            PARSER.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, NamedObjectHolder::keepNamedInOrder,
+            PARSER.declareNamedObject(NamedObjectHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
+            PARSER.declareString(NamedObjectHolder::setBar, new ParseField("bar"));
+        }
+
+        private NamedObject named;
+        private String bar;
+
+        public void setNamed(NamedObject named) {
+            this.named = named;
+        }
+
+        public void setBar(String bar) {
+            this.bar = bar;
+        }
+    }
+
+    // plural
+    static class NamedObjectsHolder {
+        public static final ObjectParser<NamedObjectsHolder, Void> PARSER = new ObjectParser<>("named_objects_holder",
+                NamedObjectsHolder::new);
+        static {
+            PARSER.declareNamedObjects(NamedObjectsHolder::setNamed, NamedObject.PARSER, NamedObjectsHolder::keepNamedInOrder,
                     new ParseField("named"));
         }
 
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index ce9dc7d574c12..6789f96d883b8 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -28,8 +28,9 @@ dependencies {
   // Upgrade to 2.10.0 or higher when jackson-core gets upgraded to 2.9.x. Blocked by #27032
   compile('com.maxmind.geoip2:geoip2:2.9.0')
   // geoip2 dependencies:
-  compile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  compile("com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}")
+  // do not hardcode this to the version in version.properties, it needs to be upgraded separately with geoip2
+  compile("com.fasterxml.jackson.core:jackson-annotations:2.8.11")
+  compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.6")
   compile('com.maxmind.db:maxmind-db:1.2.2')
 
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
deleted file mode 100644
index 5203969bcf5c0..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..19c383c96f0a0
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+1219c9aca51a37ea3e22cf88ad2e8745d1a6e02f
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index d892076210e87..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-48cb44f1dc8d3368d70581ffdbeab98ac5f5167f
\ No newline at end of file
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
index fe8e13d7ee239..dfc0f3d0228b2 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
@@ -72,7 +72,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
             searchTemplateRequest = SearchTemplateRequest.fromXContent(parser);
         }
         searchTemplateRequest.setRequest(searchRequest);
-        RestSearchAction.checkRestTotalHits(request, searchRequest);
 
         return channel -> client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel));
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
index a78cfbd2b6eec..586deae9b7309 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
@@ -32,11 +32,13 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -110,8 +112,27 @@ static SearchRequest convert(SearchTemplateRequest searchTemplateRequest, Search
             builder.parseXContent(parser, false);
             builder.explain(searchTemplateRequest.isExplain());
             builder.profile(searchTemplateRequest.isProfile());
+            checkRestTotalHitsAsInt(searchRequest, builder);
             searchRequest.source(builder);
         }
         return searchRequest;
     }
+
+    private static void checkRestTotalHitsAsInt(SearchRequest searchRequest, SearchSourceBuilder searchSourceBuilder) {
+        if (searchRequest.source() == null) {
+            searchRequest.source(new SearchSourceBuilder());
+        }
+        Integer trackTotalHitsUpTo = searchRequest.source().trackTotalHitsUpTo();
+        // trackTotalHitsUpTo is set to Integer.MAX_VALUE when `rest_total_hits_as_int` is true
+        if (trackTotalHitsUpTo != null) {
+            if (searchSourceBuilder.trackTotalHitsUpTo() == null) {
+                // trackTotalHitsUpTo should be set here, ensure that we can get an accurate total hits count
+                searchSourceBuilder.trackTotalHitsUpTo(trackTotalHitsUpTo);
+            } else if (searchSourceBuilder.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_ACCURATE
+                && searchSourceBuilder.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
+                throw new IllegalArgumentException("[" + RestSearchAction.TOTAL_HITS_AS_INT_PARAM + "] cannot be used " +
+                    "if the tracking of total hits is not accurate, got " + searchSourceBuilder.trackTotalHitsUpTo());
+            }
+        }
+    }
 }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
index f4d4f3a97e5e0..a9d3c2da68617 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
@@ -40,6 +40,19 @@
 
   - match: { hits.total: 2 }
 
+---
+"Test with invalid track_total_hits":
+
+  - do:
+      catch:              bad_request
+      search_template:
+        rest_total_hits_as_int: true
+        body: { "source" : { "query": { "match_{{template}}": {} }, "track_total_hits": "{{trackTotalHits}}" }, "params" : { "template" : "all", "trackTotalHits" : 1 } }
+
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" }
+
 ---
 "Missing template search request":
 
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
index 8f72583b61d7c..e92e10b9ad276 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
@@ -94,6 +94,11 @@ setup:
           - source: '{"query": {"{{query_type}}": {} }' # Unknown query type
             params:
               query_type: "unknown"
+          # Search 4 has an unsupported track_total_hits
+          - index: index_*
+          - source: '{"query": {"match_all": {} }, "track_total_hits": "{{trackTotalHits}}" }'
+            params:
+              trackTotalHits: 1
 
   - match:  { responses.0.hits.total:     2  }
   - match:  { responses.1.error.root_cause.0.type: json_e_o_f_exception }
@@ -101,6 +106,9 @@ setup:
   - match:  { responses.2.hits.total:     1  }
   - match:  { responses.3.error.root_cause.0.type: parsing_exception }
   - match:  { responses.3.error.root_cause.0.reason: "/unknown.query.\\[unknown\\]/" }
+  - match:  { responses.4.error.root_cause.0.type: illegal_argument_exception }
+  - match:  { responses.4.error.root_cause.0.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" }
+
 
 ---
 "Multi-search template with invalid request":
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
index 828e26e56b49d..f6d0442c0110d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
@@ -27,37 +27,39 @@
  */
 public enum Operation {
 
-    MUL     ( "*"   ),
-    DIV     ( "/"   ),
-    REM     ( "%"   ),
-    ADD     ( "+"   ),
-    SUB     ( "-"   ),
-    FIND    ( "=~"  ),
-    MATCH   ( "==~" ),
-    LSH     ( "<<"  ),
-    RSH     ( ">>"  ),
-    USH     ( ">>>" ),
-    BWNOT   ( "~"   ),
-    BWAND   ( "&"   ),
-    XOR     ( "^"   ),
-    BWOR    ( "|"   ),
-    NOT     ( "!"   ),
-    AND     ( "&&"  ),
-    OR      ( "||"  ),
-    LT      ( "<"   ),
-    LTE     ( "<="  ),
-    GT      ( ">"   ),
-    GTE     ( ">="  ),
-    EQ      ( "=="  ),
-    EQR     ( "===" ),
-    NE      ( "!="  ),
-    NER     ( "!==" ),
-    INCR    ( "++"  ),
-    DECR    ( "--"  );
+    MUL     ( "*"   , "multiplication"         ),
+    DIV     ( "/"   , "division"               ),
+    REM     ( "%"   , "remainder"              ),
+    ADD     ( "+"   , "addition"               ),
+    SUB     ( "-"   , "subtraction"            ),
+    FIND    ( "=~"  , "find"                   ),
+    MATCH   ( "==~" , "match"                  ),
+    LSH     ( "<<"  , "left shift"             ),
+    RSH     ( ">>"  , "right shift"            ),
+    USH     ( ">>>" , "unsigned shift"         ),
+    BWNOT   ( "~"   , "bitwise not"            ),
+    BWAND   ( "&"   , "bitwise and"            ),
+    XOR     ( "^"   , "bitwise xor"            ),
+    BWOR    ( "|"   , "boolean or"             ),
+    NOT     ( "!"   , "boolean not"            ),
+    AND     ( "&&"  , "boolean and"            ),
+    OR      ( "||"  , "boolean or"             ),
+    LT      ( "<"   , "less than"              ),
+    LTE     ( "<="  , "less than or equals"    ),
+    GT      ( ">"   , "greater than"           ),
+    GTE     ( ">="  , "greater than or equals" ),
+    EQ      ( "=="  , "equals"                 ),
+    EQR     ( "===" , "reference equals"       ),
+    NE      ( "!="  , "not equals"             ),
+    NER     ( "!==" , "reference not equals"   ),
+    INCR    ( "++"  , "increment"              ),
+    DECR    ( "--"  , "decrement"              );
 
     public final String symbol;
+    public final String name;
 
-    Operation(final String symbol) {
+    Operation(final String symbol, final String name) {
         this.symbol = symbol;
+        this.name = name;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 8d35968386f32..189cb94dcd70b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -35,6 +35,52 @@
  */
 public abstract class AExpression extends ANode {
 
+    public static class Input {
+
+        /**
+         * Set to false when an expression will not be read from such as
+         * a basic assignment.  Note this variable is always set by the parent
+         * as input.
+         */
+        boolean read = true;
+
+        /**
+         * Set to the expected type this node needs to be.  Note this variable
+         * is always set by the parent as input and should never be read from.
+         */
+        Class<?> expected = null;
+
+        /**
+         * Set by {@link EExplicit} if a cast made on an expression node should be
+         * explicit.
+         */
+        boolean explicit = false;
+
+        /**
+         * Set to true if a cast is allowed to boxed/unboxed.  This is used
+         * for method arguments because casting may be required.
+         */
+        boolean internal = false;
+    }
+
+    public static class Output {
+
+        /**
+         * Set to true when an expression can be considered a stand alone
+         * statement.  Used to prevent extraneous bytecode. This is always
+         * set by the node as output.
+         */
+        boolean statement = false;
+
+        /**
+         * Set to the actual type this node is.  Note this variable is always
+         * set by the node as output and should only be read from outside of the
+         * node itself.  <b>Also, actual can always be read after a cast is
+         * called on this node to get the type of the node after the cast.</b>
+         */
+        Class<?> actual = null;
+    }
+
     /**
      * Prefix is the predecessor to this node in a variable chain.
      * This is used to analyze and write variable chains in a
@@ -44,50 +90,14 @@ public abstract class AExpression extends ANode {
      */
     AExpression prefix;
 
-    /**
-     * Set to false when an expression will not be read from such as
-     * a basic assignment.  Note this variable is always set by the parent
-     * as input.
-     */
-    boolean read = true;
-
-    /**
-     * Set to true when an expression can be considered a stand alone
-     * statement.  Used to prevent extraneous bytecode. This is always
-     * set by the node as output.
-     */
-    boolean statement = false;
-
-    /**
-     * Set to the expected type this node needs to be.  Note this variable
-     * is always set by the parent as input and should never be read from.
-     */
-    Class<?> expected = null;
-
-    /**
-     * Set to the actual type this node is.  Note this variable is always
-     * set by the node as output and should only be read from outside of the
-     * node itself.  <b>Also, actual can always be read after a cast is
-     * called on this node to get the type of the node after the cast.</b>
-     */
-    Class<?> actual = null;
-
-    /**
-     * Set by {@link EExplicit} if a cast made on an expression node should be
-     * explicit.
-     */
-    boolean explicit = false;
-
-    /**
-     * Set to true if a cast is allowed to boxed/unboxed.  This is used
-     * for method arguments because casting may be required.
-     */
-    boolean internal = false;
-
-    // This is used to support the transition from a mutable to immutable state.
+    // TODO: remove placeholders once analysis and write are combined into build
+    // TODO: https://github.com/elastic/elasticsearch/issues/53561
+    // This are used to support the transition from a mutable to immutable state.
     // Currently, the IR tree is built during the user tree "write" phase, so
-    // this is stored on the node to set during the "semantic" phase and then
+    // these are stored on the node to set during the "semantic" phase and then
     // use during the "write" phase.
+    Input input = null;
+    Output output = null;
     PainlessCast cast = null;
 
     /**
@@ -111,7 +121,9 @@ public abstract class AExpression extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(ScriptRoot scriptRoot, Scope scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        throw new UnsupportedOperationException();
+    }
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
@@ -119,7 +131,7 @@ public abstract class AExpression extends ANode {
     abstract ExpressionNode write(ClassNode classNode);
 
     void cast() {
-        cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit, internal);
+        cast = AnalyzerCaster.getLegalCast(location, output.actual, input.expected, input.explicit, input.internal);
     }
 
     ExpressionNode cast(ExpressionNode expressionNode) {
@@ -129,7 +141,7 @@ ExpressionNode cast(ExpressionNode expressionNode) {
 
         CastNode castNode = new CastNode();
         castNode.setLocation(location);
-        castNode.setExpressionType(expected);
+        castNode.setExpressionType(cast.targetType);
         castNode.setCast(cast);
         castNode.setChildNode(expressionNode);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index 3ad2b0b8a0800..d8d12d8a14698 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -30,66 +30,77 @@
  */
 public abstract class AStatement extends ANode {
 
-    /**
-     * Set to true when the final statement in an {@link SClass} is reached.
-     * Used to determine whether or not an auto-return is necessary.
-     */
-    boolean lastSource = false;
-
-    /**
-     * Set to true when a loop begins.  Used by {@link SBlock} to help determine
-     * when the final statement of a loop is reached.
-     */
-    boolean beginLoop = false;
-
-    /**
-     * Set to true when inside a loop.  Used by {@link SBreak} and {@link SContinue}
-     * to determine if a break/continue statement is legal.
-     */
-    boolean inLoop = false;
-
-    /**
-     * Set to true when on the last statement of a loop.  Used by {@link SContinue}
-     * to prevent extraneous continue statements.
-     */
-    boolean lastLoop = false;
-
-    /**
-     * Set to true if a statement would cause the method to exit.  Used to
-     * determine whether or not an auto-return is necessary.
-     */
-    boolean methodEscape = false;
-
-    /**
-     * Set to true if a statement would cause a loop to exit.  Used to
-     * prevent unreachable statements.
-     */
-    boolean loopEscape = false;
-
-    /**
-     * Set to true if all current paths escape from the current {@link SBlock}.
-     * Used during the analysis phase to prevent unreachable statements and
-     * the writing phase to prevent extraneous bytecode gotos from being written.
-     */
-    boolean allEscape = false;
-
-    /**
-     * Set to true if any continue statement occurs in a loop.  Used to prevent
-     * unnecessary infinite loops.
-     */
-    boolean anyContinue = false;
+    public static class Input {
+
+        /**
+         * Set to true when the final statement in an {@link SClass} is reached.
+         * Used to determine whether or not an auto-return is necessary.
+         */
+        boolean lastSource = false;
+
+        /**
+         * Set to true when a loop begins.  Used by {@link SBlock} to help determine
+         * when the final statement of a loop is reached.
+         */
+        boolean beginLoop = false;
+
+        /**
+         * Set to true when inside a loop.  Used by {@link SBreak} and {@link SContinue}
+         * to determine if a break/continue statement is legal.
+         */
+        boolean inLoop = false;
+
+        /**
+         * Set to true when on the last statement of a loop.  Used by {@link SContinue}
+         * to prevent extraneous continue statements.
+         */
+        boolean lastLoop = false;
+    }
 
-    /**
-     * Set to true if any break statement occurs in a loop.  Used to prevent
-     * extraneous loops.
-     */
-    boolean anyBreak = false;
+    public static class Output {
+
+        /**
+         * Set to true if a statement would cause the method to exit.  Used to
+         * determine whether or not an auto-return is necessary.
+         */
+        boolean methodEscape = false;
+
+        /**
+         * Set to true if a statement would cause a loop to exit.  Used to
+         * prevent unreachable statements.
+         */
+        boolean loopEscape = false;
+
+        /**
+         * Set to true if all current paths escape from the current {@link SBlock}.
+         * Used during the analysis phase to prevent unreachable statements and
+         * the writing phase to prevent extraneous bytecode gotos from being written.
+         */
+        boolean allEscape = false;
+
+        /**
+         * Set to true if any continue statement occurs in a loop.  Used to prevent
+         * unnecessary infinite loops.
+         */
+        boolean anyContinue = false;
+
+        /**
+         * Set to true if any break statement occurs in a loop.  Used to prevent
+         * extraneous loops.
+         */
+        boolean anyBreak = false;
+
+        /**
+         * Set to the approximate number of statements in a loop block to prevent
+         * infinite loops during runtime.
+         */
+        int statementCount = 0;
+    }
 
-    /**
-     * Set to the approximate number of statements in a loop block to prevent
-     * infinite loops during runtime.
-     */
-    int statementCount = 0;
+    // TODO: remove placeholders once analysis and write are combined into build
+    // TODO: https://github.com/elastic/elasticsearch/issues/53561
+    Input input;
+    Output output;
 
     /**
      * Standard constructor with location used for error tracking.
@@ -101,7 +112,9 @@ public abstract class AStatement extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(ScriptRoot scriptRoot, Scope scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        throw new UnsupportedOperationException();
+    }
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
index 7f5beda267959..cc7262f670a94 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
 
@@ -28,11 +30,14 @@
  */
 abstract class AStoreable extends AExpression {
 
-    /**
-     * Set to true when this node is an lhs-expression and will be storing
-     * a value from an rhs-expression.
-     */
-    boolean write = false;
+    public static class Input extends AExpression.Input {
+
+        /**
+         * Set to true when this node is an lhs-expression and will be storing
+         * a value from an rhs-expression.
+         */
+        boolean write = false;
+    }
 
     /**
      * Standard constructor with location used for error tracking.
@@ -52,6 +57,10 @@ abstract class AStoreable extends AExpression {
         this.prefix = Objects.requireNonNull(prefix);
     }
 
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * Returns true if this node or a sub-node of this node can be optimized with
      * rhs actual type to avoid an unnecessary cast.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
index 4de8bdcb9a1e1..e995d637520c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
@@ -62,32 +62,24 @@ public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        analyzeLHS(scriptRoot, scope);
-        analyzeIncrDecr();
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
 
-        if (operation != null) {
-            analyzeCompound(scriptRoot, scope);
-        } else if (rhs != null) {
-            analyzeSimple(scriptRoot, scope);
-        } else {
-            throw new IllegalStateException("Illegal tree structure.");
-        }
-    }
+        Output leftOutput;
+        Output rightOutput;
 
-    private void analyzeLHS(ScriptRoot scriptRoot, Scope scope) {
         if (lhs instanceof AStoreable) {
             AStoreable lhs = (AStoreable)this.lhs;
+            AStoreable.Input leftInput = new AStoreable.Input();
 
-            lhs.read = read;
-            lhs.write = true;
-            lhs.analyze(scriptRoot, scope);
+            leftInput.read = input.read;
+            leftInput.write = true;
+            leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
         } else {
             throw new IllegalArgumentException("Left-hand side cannot be assigned a value.");
         }
-    }
 
-    private void analyzeIncrDecr() {
         if (pre && post) {
             throw createError(new IllegalStateException("Illegal tree structure."));
         } else if (pre || post) {
@@ -96,11 +88,11 @@ private void analyzeIncrDecr() {
             }
 
             if (operation == Operation.INCR) {
-                if (lhs.actual == double.class) {
+                if (leftOutput.actual == double.class) {
                     rhs = new EConstant(location, 1D);
-                } else if (lhs.actual == float.class) {
+                } else if (leftOutput.actual == float.class) {
                     rhs = new EConstant(location, 1F);
-                } else if (lhs.actual == long.class) {
+                } else if (leftOutput.actual == long.class) {
                     rhs = new EConstant(location, 1L);
                 } else {
                     rhs = new EConstant(location, 1);
@@ -108,11 +100,11 @@ private void analyzeIncrDecr() {
 
                 operation = Operation.ADD;
             } else if (operation == Operation.DECR) {
-                if (lhs.actual == double.class) {
+                if (leftOutput.actual == double.class) {
                     rhs = new EConstant(location, 1D);
-                } else if (lhs.actual == float.class) {
+                } else if (leftOutput.actual == float.class) {
                     rhs = new EConstant(location, 1F);
-                } else if (lhs.actual == long.class) {
+                } else if (leftOutput.actual == long.class) {
                     rhs = new EConstant(location, 1L);
                 } else {
                     rhs = new EConstant(location, 1);
@@ -123,103 +115,105 @@ private void analyzeIncrDecr() {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-    }
 
-    private void analyzeCompound(ScriptRoot scriptRoot, Scope scope) {
-        rhs.analyze(scriptRoot, scope);
-        boolean shift = false;
-
-        if (operation == Operation.MUL) {
-            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
-        } else if (operation == Operation.DIV) {
-            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
-        } else if (operation == Operation.REM) {
-            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
-        } else if (operation == Operation.ADD) {
-            promote = AnalyzerCaster.promoteAdd(lhs.actual, rhs.actual);
-        } else if (operation == Operation.SUB) {
-            promote = AnalyzerCaster.promoteNumeric(lhs.actual, rhs.actual, true);
-        } else if (operation == Operation.LSH) {
-            promote = AnalyzerCaster.promoteNumeric(lhs.actual, false);
-            shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false);
-            shift = true;
-        } else if (operation == Operation.RSH) {
-            promote = AnalyzerCaster.promoteNumeric(lhs.actual, false);
-            shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false);
-            shift = true;
-        } else if (operation == Operation.USH) {
-            promote = AnalyzerCaster.promoteNumeric(lhs.actual, false);
-            shiftDistance = AnalyzerCaster.promoteNumeric(rhs.actual, false);
-            shift = true;
-        } else if (operation == Operation.BWAND) {
-            promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual);
-        } else if (operation == Operation.XOR) {
-            promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual);
-        } else if (operation == Operation.BWOR) {
-            promote = AnalyzerCaster.promoteXor(lhs.actual, rhs.actual);
-        } else {
-            throw createError(new IllegalStateException("Illegal tree structure."));
-        }
+        if (operation != null) {
+            rightOutput = rhs.analyze(scriptRoot, scope, new Input());
+            boolean shift = false;
+
+            if (operation == Operation.MUL) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
+            } else if (operation == Operation.DIV) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
+            } else if (operation == Operation.REM) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
+            } else if (operation == Operation.ADD) {
+                promote = AnalyzerCaster.promoteAdd(leftOutput.actual, rightOutput.actual);
+            } else if (operation == Operation.SUB) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
+            } else if (operation == Operation.LSH) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
+                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
+                shift = true;
+            } else if (operation == Operation.RSH) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
+                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
+                shift = true;
+            } else if (operation == Operation.USH) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
+                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
+                shift = true;
+            } else if (operation == Operation.BWAND) {
+                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
+            } else if (operation == Operation.XOR) {
+                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
+            } else if (operation == Operation.BWOR) {
+                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
+            } else {
+                throw createError(new IllegalStateException("Illegal tree structure."));
+            }
 
-        if (promote == null || (shift && shiftDistance == null)) {
-            throw createError(new ClassCastException("Cannot apply compound assignment " +
-                "[" + operation.symbol + "=] to types [" + lhs.actual + "] and [" + rhs.actual + "]."));
-        }
+            if (promote == null || (shift && shiftDistance == null)) {
+                throw createError(new ClassCastException("Cannot apply compound assignment " +
+                        "[" + operation.symbol + "=] to types [" + leftOutput.actual + "] and [" + rightOutput.actual + "]."));
+            }
 
-        cat = operation == Operation.ADD && promote == String.class;
+            cat = operation == Operation.ADD && promote == String.class;
 
-        if (cat) {
-            if (rhs instanceof EBinary && ((EBinary)rhs).operation == Operation.ADD && rhs.actual == String.class) {
-                ((EBinary)rhs).cat = true;
+            if (cat) {
+                if (rhs instanceof EBinary && ((EBinary)rhs).operation == Operation.ADD && rightOutput.actual == String.class) {
+                    ((EBinary)rhs).cat = true;
+                }
             }
 
-            rhs.expected = rhs.actual;
-        } else if (shift) {
-            if (promote == def.class) {
-                // shifts are promoted independently, but for the def type, we need object.
-                rhs.expected = promote;
-            } else if (shiftDistance == long.class) {
-                rhs.expected = int.class;
-                rhs.explicit = true;
+            if (shift) {
+                if (promote == def.class) {
+                    // shifts are promoted independently, but for the def type, we need object.
+                    rhs.input.expected = promote;
+                } else if (shiftDistance == long.class) {
+                    rhs.input.expected = int.class;
+                    rhs.input.explicit = true;
+                } else {
+                    rhs.input.expected = shiftDistance;
+                }
             } else {
-                rhs.expected = shiftDistance;
+                rhs.input.expected = promote;
             }
-        } else {
-            rhs.expected = promote;
-        }
 
-        rhs.cast();
+            rhs.cast();
 
-        there = AnalyzerCaster.getLegalCast(location, lhs.actual, promote, false, false);
-        back = AnalyzerCaster.getLegalCast(location, promote, lhs.actual, true, false);
+            there = AnalyzerCaster.getLegalCast(location, leftOutput.actual, promote, false, false);
+            back = AnalyzerCaster.getLegalCast(location, promote, leftOutput.actual, true, false);
 
-        this.statement = true;
-        this.actual = read ? lhs.actual : void.class;
-    }
 
-    private void analyzeSimple(ScriptRoot scriptRoot, Scope scope) {
-        AStoreable lhs = (AStoreable)this.lhs;
+        } else if (rhs != null) {
+            AStoreable lhs = (AStoreable)this.lhs;
 
-        // If the lhs node is a def optimized node we update the actual type to remove the need for a cast.
-        if (lhs.isDefOptimized()) {
-            rhs.analyze(scriptRoot, scope);
+            // If the lhs node is a def optimized node we update the actual type to remove the need for a cast.
+            if (lhs.isDefOptimized()) {
+                rightOutput = rhs.analyze(scriptRoot, scope, new Input());
 
-            if (rhs.actual == void.class) {
-                throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment."));
+                if (rightOutput.actual == void.class) {
+                    throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment."));
+                }
+
+                rhs.input.expected = rightOutput.actual;
+                lhs.updateActual(rightOutput.actual);
+            // Otherwise, we must adapt the rhs type to the lhs type with a cast.
+            } else {
+                Input rightInput = new Input();
+                rightInput.expected = leftOutput.actual;
+                rhs.analyze(scriptRoot, scope, rightInput);
             }
 
-            rhs.expected = rhs.actual;
-            lhs.updateActual(rhs.actual);
-        // Otherwise, we must adapt the rhs type to the lhs type with a cast.
+            rhs.cast();
         } else {
-            rhs.expected = lhs.actual;
-            rhs.analyze(scriptRoot, scope);
+            throw new IllegalStateException("Illegal tree structure.");
         }
 
-        rhs.cast();
+        output.statement = true;
+        output.actual = input.read ? leftOutput.actual : void.class;
 
-        this.statement = true;
-        this.actual = read ? lhs.actual : void.class;
+        return output;
     }
 
     /**
@@ -236,12 +230,12 @@ AssignmentNode write(ClassNode classNode) {
         assignmentNode.setRightNode(rhs.cast(rhs.write(classNode)));
 
         assignmentNode.setLocation(location);
-        assignmentNode.setExpressionType(actual);
+        assignmentNode.setExpressionType(output.actual);
         assignmentNode.setCompoundType(promote);
         assignmentNode.setPre(pre);
         assignmentNode.setPost(post);
         assignmentNode.setOperation(operation);
-        assignmentNode.setRead(read);
+        assignmentNode.setRead(input.read);
         assignmentNode.setCat(cat);
         assignmentNode.setThere(there);
         assignmentNode.setBack(back);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index b5584c4b19360..5a65d74d4158c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -55,415 +55,87 @@ public EBinary(Location location, Operation operation, AExpression left, AExpres
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        originallyExplicit = explicit;
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
 
-        if (operation == Operation.MUL) {
-            analyzeMul(scriptRoot, scope);
-        } else if (operation == Operation.DIV) {
-            analyzeDiv(scriptRoot, scope);
-        } else if (operation == Operation.REM) {
-            analyzeRem(scriptRoot, scope);
-        } else if (operation == Operation.ADD) {
-            analyzeAdd(scriptRoot, scope);
-        } else if (operation == Operation.SUB) {
-            analyzeSub(scriptRoot, scope);
-        } else if (operation == Operation.FIND) {
-            analyzeRegexOp(scriptRoot, scope);
-        } else if (operation == Operation.MATCH) {
-            analyzeRegexOp(scriptRoot, scope);
-        } else if (operation == Operation.LSH) {
-            analyzeLSH(scriptRoot, scope);
-        } else if (operation == Operation.RSH) {
-            analyzeRSH(scriptRoot, scope);
-        } else if (operation == Operation.USH) {
-            analyzeUSH(scriptRoot, scope);
-        } else if (operation == Operation.BWAND) {
-            analyzeBWAnd(scriptRoot, scope);
-        } else if (operation == Operation.XOR) {
-            analyzeXor(scriptRoot, scope);
-        } else if (operation == Operation.BWOR) {
-            analyzeBWOr(scriptRoot, scope);
-        } else {
-            throw createError(new IllegalStateException("Illegal tree structure."));
-        }
-    }
-
-    private void analyzeMul(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply multiply [*] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote;
-
-        if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
-        }
-
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeDiv(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply divide [/] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote;
-
-        if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
-        }
-
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeRem(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply remainder [%] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote;
-
-        if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
-        }
-
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeAdd(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply add [+] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote;
-
-        if (promote == String.class) {
-            left.expected = left.actual;
-
-            if (left instanceof EBinary && ((EBinary)left).operation == Operation.ADD && left.actual == String.class) {
-                ((EBinary)left).cat = true;
-            }
-
-            right.expected = right.actual;
-
-            if (right instanceof EBinary && ((EBinary)right).operation == Operation.ADD && right.actual == String.class) {
-                ((EBinary)right).cat = true;
-            }
-        } else if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
-        }
-
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeSub(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply subtract [-] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote;
-
-        if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
-        }
-
-        left.cast();
-        right.cast();
-    }
+        originallyExplicit = input.explicit;
 
-    private void analyzeRegexOp(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
+        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
+        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
 
-        left.expected = String.class;
-        right.expected = Pattern.class;
-
-        left.cast();
-        right.cast();
-
-        promote = boolean.class;
-        actual = boolean.class;
-    }
-
-    private void analyzeLSH(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        Class<?> lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
-        Class<?> rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
-
-        if (lhspromote == null || rhspromote == null) {
-            throw createError(new ClassCastException("Cannot apply left shift [<<] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote = lhspromote;
-        shiftDistance = rhspromote;
-
-        if (lhspromote == def.class || rhspromote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
-            }
+        if (operation == Operation.FIND || operation == Operation.MATCH) {
+            left.input.expected = String.class;
+            right.input.expected = Pattern.class;
+            promote = boolean.class;
+            output.actual = boolean.class;
         } else {
-            left.expected = lhspromote;
-
-            if (rhspromote == long.class) {
-                right.expected = int.class;
-                right.explicit = true;
+            if (operation == Operation.MUL || operation == Operation.DIV || operation == Operation.REM || operation == Operation.SUB) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
+            } else if (operation == Operation.ADD) {
+                promote = AnalyzerCaster.promoteAdd(leftOutput.actual, rightOutput.actual);
+            } else if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, false);
+                shiftDistance = AnalyzerCaster.promoteNumeric(rightOutput.actual, false);
+
+                if (shiftDistance == null) {
+                    promote = null;
+                }
+            } else if (operation == Operation.BWOR || operation == Operation.BWAND) {
+                promote = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, false);
+            } else if (operation == Operation.XOR) {
+                promote = AnalyzerCaster.promoteXor(leftOutput.actual, rightOutput.actual);
             } else {
-                right.expected = rhspromote;
+                throw createError(new IllegalStateException("unexpected binary operation [" + operation.name + "]"));
             }
-        }
-
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeRSH(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
 
-        Class<?> lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
-        Class<?> rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
-
-        if (lhspromote == null || rhspromote == null) {
-            throw createError(new ClassCastException("Cannot apply right shift [>>] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote = lhspromote;
-        shiftDistance = rhspromote;
-
-        if (lhspromote == def.class || rhspromote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
+            if (promote == null) {
+                throw createError(new ClassCastException("cannot apply the " + operation.name + " operator " +
+                        "[" + operation.symbol + "] to the types " +
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(leftOutput.actual) + "] and " +
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
             }
-        } else {
-            left.expected = lhspromote;
 
-            if (rhspromote == long.class) {
-                right.expected = int.class;
-                right.explicit = true;
-            } else {
-                right.expected = rhspromote;
-            }
-        }
+            output.actual = promote;
 
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeUSH(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
+            if (operation == Operation.ADD && promote == String.class) {
+                left.input.expected = leftOutput.actual;
+                right.input.expected = rightOutput.actual;
 
-        Class<?> lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
-        Class<?> rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
+                if (left instanceof EBinary && ((EBinary) left).operation == Operation.ADD && leftOutput.actual == String.class) {
+                    ((EBinary) left).cat = true;
+                }
 
-        actual = promote = lhspromote;
-        shiftDistance = rhspromote;
-
-        if (lhspromote == null || rhspromote == null) {
-            throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        if (lhspromote == def.class || rhspromote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = lhspromote;
+                if (right instanceof EBinary && ((EBinary) right).operation == Operation.ADD && rightOutput.actual == String.class) {
+                    ((EBinary) right).cat = true;
+                }
+            } else if (promote == def.class || shiftDistance != null && shiftDistance == def.class) {
+                left.input.expected = leftOutput.actual;
+                right.input.expected = rightOutput.actual;
 
-            if (rhspromote == long.class) {
-                right.expected = int.class;
-                right.explicit = true;
+                if (input.expected != null) {
+                    output.actual = input.expected;
+                }
             } else {
-                right.expected = rhspromote;
-            }
-        }
-
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeBWAnd(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply and [&] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote;
-
-        if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
-        }
-
-        left.cast();
-        right.cast();
-    }
-
-    private void analyzeXor(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply xor [^] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
+                left.input.expected = promote;
 
-        actual = promote;
-
-        if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-            if (expected != null) {
-                actual = expected;
+                if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
+                    if (shiftDistance == long.class) {
+                        right.input.expected = int.class;
+                        right.input.explicit = true;
+                    } else {
+                        right.input.expected = shiftDistance;
+                    }
+                } else {
+                    right.input.expected = promote;
+                }
             }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
         }
 
         left.cast();
         right.cast();
-    }
-
-    private void analyzeBWOr(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply or [|] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        actual = promote;
-
-        if (promote == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-            if (expected != null) {
-                actual = expected;
-            }
-        } else {
-            left.expected = promote;
-            right.expected = promote;
-        }
-
-        left.cast();
-        right.cast();
+        return output;
     }
 
     @Override
@@ -474,7 +146,7 @@ BinaryMathNode write(ClassNode classNode) {
         binaryMathNode.setRightNode(right.cast(right.write(classNode)));
 
         binaryMathNode.setLocation(location);
-        binaryMathNode.setExpressionType(actual);
+        binaryMathNode.setExpressionType(output.actual);
         binaryMathNode.setBinaryType(promote);
         binaryMathNode.setShiftType(shiftDistance);
         binaryMathNode.setOperation(operation);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index c7538ffa7a887..f2acae9299f20 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -46,16 +46,23 @@ public EBool(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        left.expected = boolean.class;
-        left.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        Input leftInput = new Input();
+        leftInput.expected = boolean.class;
+        left.analyze(scriptRoot, scope, leftInput);
         left.cast();
 
-        right.expected = boolean.class;
-        right.analyze(scriptRoot, scope);
+        Input rightInput = new Input();
+        rightInput.expected = boolean.class;
+        right.analyze(scriptRoot, scope, rightInput);
         right.cast();
 
-        actual = boolean.class;
+        output.actual = boolean.class;
+
+        return output;
     }
 
     @Override
@@ -66,7 +73,7 @@ BooleanNode write(ClassNode classNode) {
         booleanNode.setRightNode(right.cast(right.write(classNode)));
 
         booleanNode.setLocation(location);
-        booleanNode.setExpressionType(actual);
+        booleanNode.setExpressionType(output.actual);
         booleanNode.setOperation(operation);
 
         return booleanNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 10d941fdce90f..6e78fb972008c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -40,19 +40,24 @@ public EBoolean(Location location, boolean constant) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
         }
 
-        actual = boolean.class;
+        output.actual = boolean.class;
+
+        return output;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(actual);
+        constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index fa2e1fce72c4f..ad4dce45d4d7a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -58,7 +58,10 @@ public ECallLocal(Location location, String name, List<AExpression> arguments) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         localFunction = scriptRoot.getFunctionTable().getFunction(name, arguments.size());
 
         // user cannot call internal functions, reset to null if an internal function is found
@@ -112,21 +115,21 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
 
         if (localFunction != null) {
             typeParameters = new ArrayList<>(localFunction.getTypeParameters());
-            actual = localFunction.getReturnType();
+            output.actual = localFunction.getReturnType();
         } else if (importedMethod != null) {
             scriptRoot.markNonDeterministic(importedMethod.annotations.containsKey(NonDeterministicAnnotation.class));
             typeParameters = new ArrayList<>(importedMethod.typeParameters);
-            actual = importedMethod.returnType;
+            output.actual = importedMethod.returnType;
         } else if (classBinding != null) {
             scriptRoot.markNonDeterministic(classBinding.annotations.containsKey(NonDeterministicAnnotation.class));
             typeParameters = new ArrayList<>(classBinding.typeParameters);
-            actual = classBinding.returnType;
+            output.actual = classBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("class_binding");
             scriptRoot.getClassNode().addField(new SField(location,
                     Modifier.PRIVATE, bindingName, classBinding.javaConstructor.getDeclaringClass()));
         } else if (instanceBinding != null) {
             typeParameters = new ArrayList<>(instanceBinding.typeParameters);
-            actual = instanceBinding.returnType;
+            output.actual = instanceBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("instance_binding");
             scriptRoot.getClassNode().addField(new SField(location, Modifier.STATIC | Modifier.PUBLIC,
                     bindingName, instanceBinding.targetInstance.getClass()));
@@ -141,13 +144,16 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = typeParameters.get(argument + classBindingOffset);
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.expected = typeParameters.get(argument + classBindingOffset);
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
-        statement = true;
+        output.statement = true;
+
+        return output;
     }
 
     @Override
@@ -159,7 +165,7 @@ MemberCallNode write(ClassNode classNode) {
         }
 
         memberCallNode.setLocation(location);
-        memberCallNode.setExpressionType(actual);
+        memberCallNode.setExpressionType(output.actual);
         memberCallNode.setLocalFunction(localFunction);
         memberCallNode.setImportedMethod(importedMethod);
         memberCallNode.setClassBinding(classBinding);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index adf67370e3b03..fa8b25cd861f9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -51,9 +51,12 @@ public ECapturingFunctionRef(Location location, String variable, String call) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         captured = scope.getVariable(location, variable);
-        if (expected == null) {
+        if (input.expected == null) {
             if (captured.getType() == def.class) {
                 // dynamic implementation
                 defPointer = "D" + variable + "." + call + ",1";
@@ -61,16 +64,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                 // typed implementation
                 defPointer = "S" + captured.getCanonicalTypeName() + "." + call + ",1";
             }
-            actual = String.class;
+            output.actual = String.class;
         } else {
             defPointer = null;
             // static case
             if (captured.getType() != def.class) {
                 ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(), location,
-                        expected, captured.getCanonicalTypeName(), call, 1);
+                        input.expected, captured.getCanonicalTypeName(), call, 1);
             }
-            actual = expected;
+            output.actual = input.expected;
         }
+
+        return output;
     }
 
     @Override
@@ -78,7 +83,7 @@ CapturingFuncRefNode write(ClassNode classNode) {
         CapturingFuncRefNode capturingFuncRefNode = new CapturingFuncRefNode();
 
         capturingFuncRefNode.setLocation(location);
-        capturingFuncRefNode.setExpressionType(actual);
+        capturingFuncRefNode.setExpressionType(output.actual);
         capturingFuncRefNode.setCapturedName(captured.getName());
         capturingFuncRefNode.setName(call);
         capturingFuncRefNode.setPointer(defPointer);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 512acee938a78..32555c18625f7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -51,240 +51,47 @@ public EComp(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (operation == Operation.EQ) {
-            analyzeEq(scriptRoot, scope);
-        } else if (operation == Operation.EQR) {
-            analyzeEqR(scriptRoot, scope);
-        } else if (operation == Operation.NE) {
-            analyzeNE(scriptRoot, scope);
-        } else if (operation == Operation.NER) {
-            analyzeNER(scriptRoot, scope);
-        } else if (operation == Operation.GTE) {
-            analyzeGTE(scriptRoot, scope);
-        } else if (operation == Operation.GT) {
-            analyzeGT(scriptRoot, scope);
-        } else if (operation == Operation.LTE) {
-            analyzeLTE(scriptRoot, scope);
-        } else if (operation == Operation.LT) {
-            analyzeLT(scriptRoot, scope);
-        } else {
-            throw createError(new IllegalStateException("Illegal tree structure."));
-        }
-    }
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
 
-    private void analyzeEq(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
-
-        if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply equals [==] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
+        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
+        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
 
-        if (promotedType == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
+        if (operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER) {
+            promotedType = AnalyzerCaster.promoteEquality(leftOutput.actual, rightOutput.actual);
+        } else if (operation == Operation.GT || operation == Operation.GTE || operation == Operation.LT || operation == Operation.LTE) {
+            promotedType = AnalyzerCaster.promoteNumeric(leftOutput.actual, rightOutput.actual, true);
         } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
-        }
-
-        if (left instanceof ENull && right instanceof ENull) {
-            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
+            throw createError(new IllegalStateException("unexpected binary operation [" + operation.name + "]"));
         }
 
-        left.cast();
-        right.cast();
-
-        actual = boolean.class;
-    }
-
-    private void analyzeEqR(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
-
         if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply reference equals [===] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        left.expected = promotedType;
-        right.expected = promotedType;
-
-        if (left instanceof ENull && right instanceof ENull) {
-            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
+            throw createError(new ClassCastException("cannot apply the " + operation.name + " operator " +
+                    "[" + operation.symbol + "] to the types " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(leftOutput.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
         }
 
-        left.cast();
-        right.cast();
-
-        actual = boolean.class;
-    }
-
-    private void analyzeNE(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
-
-        if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply not equals [!=] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        if (promotedType == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
+        if (operation != Operation.EQR && operation != Operation.NER && promotedType == def.class) {
+            left.input.expected = leftOutput.actual;
+            right.input.expected = rightOutput.actual;
         } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
+            left.input.expected = promotedType;
+            right.input.expected = promotedType;
         }
 
-        if (left instanceof ENull && right instanceof ENull) {
-            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
+        if ((operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER)
+                && left instanceof ENull && right instanceof ENull) {
+            throw createError(new IllegalArgumentException("extraneous comparison of [null] constants"));
         }
 
         left.cast();
         right.cast();
 
-        actual = boolean.class;
-    }
-
-    private void analyzeNER(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
-
-        if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        left.expected = promotedType;
-        right.expected = promotedType;
-
-        if (left instanceof ENull && right instanceof ENull) {
-            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
-        }
-
-        left.cast();
-        right.cast();
-
-        actual = boolean.class;
-    }
-
-    private void analyzeGTE(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        if (promotedType == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-        } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
-        }
-
-        left.cast();
-        right.cast();
-
-        actual = boolean.class;
-    }
-
-    private void analyzeGT(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply greater than [>] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        if (promotedType == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-        } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
-        }
-
-        left.cast();
-        right.cast();
-
-        actual = boolean.class;
-    }
-
-    private void analyzeLTE(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        if (promotedType == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-        } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
-        }
-
-        left.cast();
-        right.cast();
-
-        actual = boolean.class;
-    }
-
-    private void analyzeLT(ScriptRoot scriptRoot, Scope variables) {
-        left.analyze(scriptRoot, variables);
-        right.analyze(scriptRoot, variables);
-
-        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
-
-        if (promotedType == null) {
-            throw createError(new ClassCastException("Cannot apply less than [>=] to types " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
-        }
-
-        if (promotedType == def.class) {
-            left.expected = left.actual;
-            right.expected = right.actual;
-        } else {
-            left.expected = promotedType;
-            right.expected = promotedType;
-        }
-
-        left.cast();
-        right.cast();
+        output.actual = boolean.class;
 
-        actual = boolean.class;
+        return output;
     }
 
     @Override
@@ -295,7 +102,7 @@ ComparisonNode write(ClassNode classNode) {
         comparisonNode.setRightNode(right.cast(right.write(classNode)));
 
         comparisonNode.setLocation(location);
-        comparisonNode.setExpressionType(actual);
+        comparisonNode.setExpressionType(output.actual);
         comparisonNode.setComparisonType(promotedType);
         comparisonNode.setOperation(operation);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index c1eb8fbe510da..d523838ecf66d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -47,38 +47,48 @@ public EConditional(Location location, AExpression condition, AExpression left,
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        condition.expected = boolean.class;
-        condition.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        Input conditionInput = new Input();
+        conditionInput.expected = boolean.class;
+        condition.analyze(scriptRoot, scope, conditionInput);
         condition.cast();
 
-        left.expected = expected;
-        left.explicit = explicit;
-        left.internal = internal;
-        right.expected = expected;
-        right.explicit = explicit;
-        right.internal = internal;
-        actual = expected;
+        Input leftInput = new Input();
+        leftInput.expected = input.expected;
+        leftInput.explicit = input.explicit;
+        leftInput.internal = input.internal;
+
+        Input rightInput = new Input();
+        rightInput.expected = input.expected;
+        rightInput.explicit = input.explicit;
+        rightInput.internal = input.internal;
 
-        left.analyze(scriptRoot, scope);
-        right.analyze(scriptRoot, scope);
+        output.actual = input.expected;
 
-        if (expected == null) {
-            Class<?> promote = AnalyzerCaster.promoteConditional(left.actual, right.actual);
+        Output leftOutput = left.analyze(scriptRoot, scope, leftInput);
+        Output rightOutput = right.analyze(scriptRoot, scope, rightInput);
+
+        if (input.expected == null) {
+            Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
 
             if (promote == null) {
-                throw createError(new ClassCastException("cannot apply a conditional operator [?:] to the types " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]"));
+                throw createError(new ClassCastException("cannot apply the conditional operator [?:] to the types " +
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(leftOutput.actual) + "] and " +
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
             }
 
-            left.expected = promote;
-            right.expected = promote;
-            actual = promote;
+            left.input.expected = promote;
+            right.input.expected = promote;
+            output.actual = promote;
         }
 
         left.cast();
         right.cast();
+
+        return output;
     }
 
     @Override
@@ -90,7 +100,7 @@ ConditionalNode write(ClassNode classNode) {
         conditionalNode.setConditionNode(condition.cast(condition.write(classNode)));
 
         conditionalNode.setLocation(location);
-        conditionalNode.setExpressionType(actual);
+        conditionalNode.setExpressionType(output.actual);
 
         return conditionalNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index c02beca4ec044..5caa6bca92895 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 /**
@@ -40,35 +41,42 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         if (constant instanceof String) {
-            actual = String.class;
+            output.actual = String.class;
         } else if (constant instanceof Double) {
-            actual = double.class;
+            output.actual = double.class;
         } else if (constant instanceof Float) {
-            actual = float.class;
+            output.actual = float.class;
         } else if (constant instanceof Long) {
-            actual = long.class;
+            output.actual = long.class;
         } else if (constant instanceof Integer) {
-            actual = int.class;
+            output.actual = int.class;
         } else if (constant instanceof Character) {
-            actual = char.class;
+            output.actual = char.class;
         } else if (constant instanceof Short) {
-            actual = short.class;
+            output.actual = short.class;
         } else if (constant instanceof Byte) {
-            actual = byte.class;
+            output.actual = byte.class;
         } else if (constant instanceof Boolean) {
-            actual = boolean.class;
+            output.actual = boolean.class;
         } else {
-            throw createError(new IllegalStateException("Illegal tree structure."));
+            throw createError(new IllegalStateException("unexpected type " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(constant.getClass()) + "] " +
+                    "for constant node"));
         }
+
+        return output;
     }
 
     @Override
     ConstantNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(actual);
+        constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 049e52799f1f7..405f90b356f1a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -44,15 +44,18 @@ public EDecimal(Location location, String value) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
         }
 
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = float.class;
+                output.actual = float.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid float constant [" + value + "]."));
             }
@@ -63,18 +66,20 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             }
             try {
                 constant = Double.parseDouble(toParse);
-                actual = double.class;
+                output.actual = double.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid double constant [" + value + "]."));
             }
         }
+
+        return output;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(actual);
+        constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
index f9ffb019c7a7e..6c771af47caf4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
@@ -44,19 +44,24 @@ public EElvis(Location location, AExpression lhs, AExpression rhs) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (expected != null && expected.isPrimitive()) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.expected != null && input.expected.isPrimitive()) {
             throw createError(new IllegalArgumentException("Elvis operator cannot return primitives"));
         }
-        lhs.expected = expected;
-        lhs.explicit = explicit;
-        lhs.internal = internal;
-        rhs.expected = expected;
-        rhs.explicit = explicit;
-        rhs.internal = internal;
-        actual = expected;
-        lhs.analyze(scriptRoot, scope);
-        rhs.analyze(scriptRoot, scope);
+        Input leftInput = new Input();
+        leftInput.expected = input.expected;
+        leftInput.explicit = input.explicit;
+        leftInput.internal = input.internal;
+        Input rightInput = new Input();
+        rightInput.expected = input.expected;
+        rightInput.explicit = input.explicit;
+        rightInput.internal = input.internal;
+        output.actual = input.expected;
+        Output leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
+        Output rightOutput = rhs.analyze(scriptRoot, scope, rightInput);
 
         if (lhs instanceof ENull) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is null."));
@@ -68,23 +73,25 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                 || lhs instanceof EConstant) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is a constant."));
         }
-        if (lhs.actual.isPrimitive()) {
+        if (leftOutput.actual.isPrimitive()) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is a primitive."));
         }
         if (rhs instanceof ENull) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. RHS is null."));
         }
 
-        if (expected == null) {
-            Class<?> promote = AnalyzerCaster.promoteConditional(lhs.actual, rhs.actual);
+        if (input.expected == null) {
+            Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
 
-            lhs.expected = promote;
-            rhs.expected = promote;
-            actual = promote;
+            lhs.input.expected = promote;
+            rhs.input.expected = promote;
+            output.actual = promote;
         }
 
         lhs.cast();
         rhs.cast();
+
+        return output;
     }
 
     @Override
@@ -95,7 +102,7 @@ ElvisNode write(ClassNode classNode) {
         elvisNode.setRightNode(rhs.cast(rhs.write(classNode)));
 
         elvisNode.setLocation(location);
-        elvisNode.setExpressionType(actual);
+        elvisNode.setExpressionType(output.actual);
 
         return elvisNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index d3d288bde8162..b76dcd946b3b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -43,17 +43,23 @@ public EExplicit(Location location, String type, AExpression child) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
 
-        if (actual == null) {
+        output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
+
+        if (output.actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
 
-        child.expected = actual;
-        child.explicit = true;
-        child.analyze(scriptRoot, scope);
+        Input childInput = new Input();
+        childInput.expected = output.actual;
+        childInput.explicit = true;
+        child.analyze(scriptRoot, scope, childInput);
         child.cast();
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 67a2e24f0b7c6..62bb34c215590 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -48,16 +48,22 @@ public EFunctionRef(Location location, String type, String call) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (expected == null) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.expected == null) {
             ref = null;
-            actual = String.class;
+            output.actual = String.class;
             defPointer = "S" + type + "." + call + ",0";
         } else {
             defPointer = null;
-            ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(), location, expected, type, call, 0);
-            actual = expected;
+            ref = FunctionRef.create(
+                    scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(), location, input.expected, type, call, 0);
+            output.actual = input.expected;
         }
+
+        return output;
     }
 
     @Override
@@ -65,7 +71,7 @@ FuncRefNode write(ClassNode classNode) {
         FuncRefNode funcRefNode = new FuncRefNode();
 
         funcRefNode.setLocation(location);
-        funcRefNode.setExpressionType(actual);
+        funcRefNode.setExpressionType(output.actual);
         funcRefNode.setFuncRef(ref);
 
         return funcRefNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index 01c789dd89e4f..e6ee1a03602b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -38,7 +38,7 @@ public final class EInstanceof extends AExpression {
     private final String type;
 
     private Class<?> resolvedType;
-    private Class<?> instanceType;
+    private Class<?> expressionType;
     private boolean primitiveExpression;
 
     public EInstanceof(Location location, AExpression expression, String type) {
@@ -48,7 +48,10 @@ public EInstanceof(Location location, AExpression expression, String type) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         // ensure the specified type is part of the definition
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
 
@@ -61,17 +64,19 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                 PainlessLookupUtility.typeToJavaType(clazz);
 
         // analyze and cast the expression
-        expression.analyze(scriptRoot, scope);
-        expression.expected = expression.actual;
+        Output expressionOutput = expression.analyze(scriptRoot, scope, new Input());
+        expression.input.expected = expressionOutput.actual;
         expression.cast();
 
         // record if the expression returns a primitive
-        primitiveExpression = expression.actual.isPrimitive();
+        primitiveExpression = expressionOutput.actual.isPrimitive();
         // map to wrapped type for primitive types
-        instanceType = expression.actual.isPrimitive() ?
-            PainlessLookupUtility.typeToBoxedType(expression.actual) : PainlessLookupUtility.typeToJavaType(clazz);
+        expressionType = expressionOutput.actual.isPrimitive() ?
+            PainlessLookupUtility.typeToBoxedType(expressionOutput.actual) : PainlessLookupUtility.typeToJavaType(clazz);
+
+        output.actual = boolean.class;
 
-        actual = boolean.class;
+        return output;
     }
 
     @Override
@@ -81,8 +86,8 @@ InstanceofNode write(ClassNode classNode) {
         instanceofNode.setChildNode(expression.cast(expression.write(classNode)));
 
         instanceofNode.setLocation(location);
-        instanceofNode.setExpressionType(actual);
-        instanceofNode.setInstanceType(instanceType);
+        instanceofNode.setExpressionType(output.actual);
+        instanceofNode.setInstanceType(expressionType);
         instanceofNode.setResolvedType(resolvedType);
         instanceofNode.setPrimitiveResult(primitiveExpression);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 5dbebcf554af3..a4cb19d048c62 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -91,11 +91,14 @@ public ELambda(Location location,
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         List<Class<?>> typeParameters = new ArrayList<>();
         PainlessMethod interfaceMethod;
         // inspect the target first, set interface method if we know it.
-        if (expected == null) {
+        if (input.expected == null) {
             interfaceMethod = null;
             // we don't know anything: treat as def
             returnType = def.class;
@@ -117,15 +120,15 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
 
         } else {
             // we know the method statically, infer return type and any unknown/def types
-            interfaceMethod = scriptRoot.getPainlessLookup().lookupFunctionalInterfacePainlessMethod(expected);
+            interfaceMethod = scriptRoot.getPainlessLookup().lookupFunctionalInterfacePainlessMethod(input.expected);
             if (interfaceMethod == null) {
                 throw createError(new IllegalArgumentException("Cannot pass lambda to " +
-                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"));
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(input.expected) + "], not a functional interface"));
             }
             // check arity before we manipulate parameters
             if (interfaceMethod.typeParameters.size() != paramTypeStrs.size())
                 throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.javaMethod.getName() +
-                        "] in [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]");
+                        "] in [" + PainlessLookupUtility.typeToCanonicalTypeName(input.expected) + "]");
             // for method invocation, its allowed to ignore the return value
             if (interfaceMethod.returnType == void.class) {
                 returnType = def.class;
@@ -162,17 +165,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         if (block.statements.isEmpty()) {
             throw createError(new IllegalArgumentException("cannot generate empty lambda"));
         }
-        block.lastSource = true;
-        block.analyze(scriptRoot, lambdaScope);
-        captures = new ArrayList<>(lambdaScope.getCaptures());
+        AStatement.Input blockInput = new AStatement.Input();
+        blockInput.lastSource = true;
+        AStatement.Output blockOutput = block.analyze(scriptRoot, lambdaScope, blockInput);
 
-        if (block.methodEscape == false) {
+        if (blockOutput.methodEscape == false) {
             throw createError(new IllegalArgumentException("not all paths return a value for lambda"));
         }
 
         maxLoopCounter = scriptRoot.getCompilerSettings().getMaxLoopCounter();
 
         // prepend capture list to lambda's arguments
+        captures = new ArrayList<>(lambdaScope.getCaptures());
         this.typeParameters = new ArrayList<>(captures.size() + typeParameters.size());
         parameterNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
@@ -187,16 +191,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         scriptRoot.getFunctionTable().addFunction(name, returnType, this.typeParameters, true, true);
 
         // setup method reference to synthetic method
-        if (expected == null) {
+        if (input.expected == null) {
             ref = null;
-            actual = String.class;
+            output.actual = String.class;
             defPointer = "Sthis." + name + "," + captures.size();
         } else {
             defPointer = null;
             ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(),
-                    location, expected, "this", name, captures.size());
-            actual = expected;
+                    location, input.expected, "this", name, captures.size());
+            output.actual = input.expected;
         }
+
+        return output;
     }
 
     @Override
@@ -220,7 +226,7 @@ LambdaNode write(ClassNode classNode) {
         LambdaNode lambdaNode = new LambdaNode();
 
         lambdaNode.setLocation(location);
-        lambdaNode.setExpressionType(actual);
+        lambdaNode.setExpressionType(output.actual);
         lambdaNode.setFuncRef(ref);
 
         for (Variable capture : captures) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index 27c6d1c7897f1..cb49bce8a11c3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -49,34 +49,40 @@ public EListInit(Location location, List<AExpression> values) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from list initializer."));
         }
 
-        actual = ArrayList.class;
+        output.actual = ArrayList.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(actual, 0);
+        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
-                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
+                    "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(actual, false, "add", 1);
+        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "add", 1);
 
         if (method == null) {
-            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", add/1] not found"));
+            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", add/1] not found"));
         }
 
         for (int index = 0; index < values.size(); ++index) {
             AExpression expression = values.get(index);
 
-            expression.expected = def.class;
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.expected = def.class;
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
+
+        return output;
     }
 
     @Override
@@ -88,7 +94,7 @@ ListInitializationNode write(ClassNode classNode) {
         }
 
         listInitializationNode.setLocation(location);
-        listInitializationNode.setExpressionType(actual);
+        listInitializationNode.setExpressionType(output.actual);
         listInitializationNode.setConstructor(constructor);
         listInitializationNode.setMethod(method);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index 69eb4e10f8866..f846a18eb89f9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -51,24 +51,27 @@ public EMapInit(Location location, List<AExpression> keys, List<AExpression> val
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from map initializer."));
         }
 
-        actual = HashMap.class;
+        output.actual = HashMap.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(actual, 0);
+        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
-                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/0] not found"));
+                    "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(actual, false, "put", 2);
+        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "put", 2);
 
         if (method == null) {
-            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(actual) + ", put/2] not found"));
+            throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", put/2] not found"));
         }
 
         if (keys.size() != values.size()) {
@@ -78,20 +81,24 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         for (int index = 0; index < keys.size(); ++index) {
             AExpression expression = keys.get(index);
 
-            expression.expected = def.class;
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.expected = def.class;
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
         for (int index = 0; index < values.size(); ++index) {
             AExpression expression = values.get(index);
 
-            expression.expected = def.class;
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.expected = def.class;
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
+
+        return output;
     }
 
     @Override
@@ -105,7 +112,7 @@ MapInitializationNode write(ClassNode classNode) {
         }
 
         mapInitializationNode.setLocation(location);
-        mapInitializationNode.setExpressionType(actual);
+        mapInitializationNode.setExpressionType(output.actual);
         mapInitializationNode.setConstructor(constructor);
         mapInitializationNode.setMethod(method);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
index 7e2a315fc24e1..73603d79e6c89 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
@@ -46,8 +46,11 @@ public ENewArray(Location location, String type, List<AExpression> arguments, bo
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
              throw createError(new IllegalArgumentException("A newly created array must be read from."));
         }
 
@@ -60,13 +63,16 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = initialize ? clazz.getComponentType() : int.class;
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.expected = initialize ? clazz.getComponentType() : int.class;
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
-        actual = clazz;
+        output.actual = clazz;
+
+        return output;
     }
 
     @Override
@@ -78,7 +84,7 @@ NewArrayNode write(ClassNode classNode) {
         }
 
         newArrayNode.setLocation(location);
-        newArrayNode.setExpressionType(actual);
+        newArrayNode.setExpressionType(output.actual);
         newArrayNode.setInitialize(initialize);
 
         return newArrayNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
index 5302f17d85907..0a132c12e5adb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
@@ -48,7 +48,10 @@ public ENewArrayFunctionRef(Location location, String type) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         SReturn code = new SReturn(location, new ENewArray(location, type, Arrays.asList(new EVariable(location, "size")), false));
         function = new SFunction(
                 location, type, scriptRoot.getNextSyntheticName("newarray"),
@@ -58,16 +61,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         function.analyze(scriptRoot);
         scriptRoot.getFunctionTable().addFunction(function.name, function.returnType, function.typeParameters, true, true);
 
-        if (expected == null) {
+        if (input.expected == null) {
             ref = null;
-            actual = String.class;
+            output.actual = String.class;
             defPointer = "Sthis." + function.name + ",0";
         } else {
             defPointer = null;
             ref = FunctionRef.create(scriptRoot.getPainlessLookup(), scriptRoot.getFunctionTable(),
-                    location, expected, "this", function.name, 0);
-            actual = expected;
+                    location, input.expected, "this", function.name, 0);
+            output.actual = input.expected;
         }
+
+        return output;
     }
 
     @Override
@@ -77,7 +82,7 @@ NewArrayFuncRefNode write(ClassNode classNode) {
         NewArrayFuncRefNode newArrayFuncRefNode = new NewArrayFuncRefNode();
 
         newArrayFuncRefNode.setLocation(location);
-        newArrayFuncRefNode.setExpressionType(actual);
+        newArrayFuncRefNode.setExpressionType(output.actual);
         newArrayFuncRefNode.setFuncRef(ref);
 
         return newArrayFuncRefNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index 0696b75696e2d..dda96482a472d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -51,18 +51,21 @@ public ENewObj(Location location, String type, List<AExpression> arguments) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
 
-        if (actual == null) {
+        output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
+
+        if (output.actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(actual, arguments.size());
+        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, arguments.size());
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
-                    "constructor [" + typeToCanonicalTypeName(actual) + ", <init>/" + arguments.size() + "] not found"));
+                    "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/" + arguments.size() + "] not found"));
         }
 
         scriptRoot.markNonDeterministic(constructor.annotations.containsKey(NonDeterministicAnnotation.class));
@@ -72,20 +75,23 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
 
         if (constructor.typeParameters.size() != arguments.size()) {
             throw createError(new IllegalArgumentException(
-                    "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " +
+                    "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(output.actual) + "] " +
                     "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "]."));
         }
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = types[argument];
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.expected = types[argument];
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
-        statement = true;
+        output.statement = true;
+
+        return output;
     }
 
     @Override
@@ -97,8 +103,8 @@ NewObjectNode write(ClassNode classNode) {
         }
 
         newObjectNode.setLocation(location);
-        newObjectNode.setExpressionType(actual);
-        newObjectNode.setRead(read);
+        newObjectNode.setExpressionType(output.actual);
+        newObjectNode.setRead(input.read);
         newObjectNode.setConstructor(constructor);
 
         return newObjectNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 6d26a7de1e79f..841bd09586bed 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -36,21 +36,26 @@ public ENull(Location location) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from null constant."));
         }
 
-        if (expected != null) {
-            if (expected.isPrimitive()) {
+        if (input.expected != null) {
+            if (input.expected.isPrimitive()) {
                 throw createError(new IllegalArgumentException(
-                    "Cannot cast null to a primitive type [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]."));
+                    "Cannot cast null to a primitive type [" + PainlessLookupUtility.typeToCanonicalTypeName(input.expected) + "]."));
             }
 
-            actual = expected;
+            output.actual = input.expected;
         } else {
-            actual = Object.class;
+            output.actual = Object.class;
         }
+
+        return output;
     }
 
     @Override
@@ -58,7 +63,7 @@ NullNode write(ClassNode classNode) {
         NullNode nullNode = new NullNode();
 
         nullNode.setLocation(location);
-        nullNode.setExpressionType(actual);
+        nullNode.setExpressionType(output.actual);
 
         return nullNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 5c689f090acaf..f7cbe681b639d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -46,8 +46,11 @@ public ENumeric(Location location, String value, int radix) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
         }
 
@@ -58,7 +61,7 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
 
             try {
                 constant = Double.parseDouble(value.substring(0, value.length() - 1));
-                actual = double.class;
+                output.actual = double.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid double constant [" + value + "]."));
             }
@@ -69,34 +72,34 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
 
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = float.class;
+                output.actual = float.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid float constant [" + value + "]."));
             }
         } else if (value.endsWith("l") || value.endsWith("L")) {
             try {
                 constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
-                actual = long.class;
+                output.actual = long.class;
             } catch (NumberFormatException exception) {
                 throw createError(new IllegalArgumentException("Invalid long constant [" + value + "]."));
             }
         } else {
             try {
-                Class<?> sort = expected == null ? int.class : expected;
+                Class<?> sort = input.expected == null ? int.class : input.expected;
                 int integer = Integer.parseInt(value, radix);
 
                 if (sort == byte.class && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) {
                     constant = (byte)integer;
-                    actual = byte.class;
+                    output.actual = byte.class;
                 } else if (sort == char.class && integer >= Character.MIN_VALUE && integer <= Character.MAX_VALUE) {
                     constant = (char)integer;
-                    actual = char.class;
+                    output.actual = char.class;
                 } else if (sort == short.class && integer >= Short.MIN_VALUE && integer <= Short.MAX_VALUE) {
                     constant = (short)integer;
-                    actual = short.class;
+                    output.actual = short.class;
                 } else {
                     constant = integer;
-                    actual = int.class;
+                    output.actual = int.class;
                 }
             } catch (NumberFormatException exception) {
                 try {
@@ -110,13 +113,15 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                 throw createError(new IllegalArgumentException("Invalid int constant [" + value + "]."));
             }
         }
+
+        return output;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(actual);
+        constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
index 6c3527291afa9..3847397407dd8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
@@ -63,14 +63,18 @@ public ERegex(Location location, String pattern, String flagsString) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+
         if (scriptRoot.getCompilerSettings().areRegexesEnabled() == false) {
             throw createError(new IllegalStateException("Regexes are disabled. Set [script.painless.regex.enabled] to [true] "
                     + "in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep "
                     + "recursion and long loops."));
         }
 
-        if (!read) {
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Regex constant may only be read [" + pattern + "]."));
         }
 
@@ -82,7 +86,9 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         name = scriptRoot.getNextSyntheticName("regex");
-        actual = Pattern.class;
+        output.actual = Pattern.class;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
index 9714fec7ce52c..0706e321e5312 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
@@ -41,12 +41,17 @@ public EStatic(Location location, String type) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
 
-        if (actual == null) {
+        output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
+
+        if (output.actual == null) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
+
+        return output;
     }
 
     @Override
@@ -54,7 +59,7 @@ StaticNode write(ClassNode classNode) {
         StaticNode staticNode = new StaticNode();
 
         staticNode.setLocation(location);
-        staticNode.setExpressionType(actual);
+        staticNode.setExpressionType(output.actual);
 
         return staticNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
index 423ff49e521b9..5cc5b67fe511b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
@@ -42,19 +42,24 @@ public EString(Location location, String string) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!read) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
         }
 
-        actual = String.class;
+        output.actual = String.class;
+
+        return output;
     }
 
     @Override
     ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
-        constantNode.setExpressionType(actual);
+        constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
         return constantNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 3fe797f0352ee..e498ceb3578c8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -51,88 +51,43 @@ public EUnary(Location location, Operation operation, AExpression child) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        originallyExplicit = explicit;
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
 
-        if (operation == Operation.NOT) {
-            analyzeNot(scriptRoot, scope);
-        } else if (operation == Operation.BWNOT) {
-            analyzeBWNot(scriptRoot, scope);
-        } else if (operation == Operation.ADD) {
-            analyzerAdd(scriptRoot, scope);
-        } else if (operation == Operation.SUB) {
-            analyzerSub(scriptRoot, scope);
-        } else {
-            throw createError(new IllegalStateException("Illegal tree structure."));
-        }
-    }
-
-    void analyzeNot(ScriptRoot scriptRoot, Scope variables) {
-        child.expected = boolean.class;
-        child.analyze(scriptRoot, variables);
-        child.cast();
-
-        actual = boolean.class;
-    }
-
-    void analyzeBWNot(ScriptRoot scriptRoot, Scope variables) {
-        child.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(child.actual, false);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply not [~] to type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(child.actual) + "]."));
-        }
-
-        child.expected = promote;
-        child.cast();
-
-        if (promote == def.class && expected != null) {
-            actual = expected;
-        } else {
-            actual = promote;
-        }
-    }
-
-    void analyzerAdd(ScriptRoot scriptRoot, Scope variables) {
-        child.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(child.actual, true);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply positive [+] to type " +
-                    "[" + PainlessLookupUtility.typeToJavaType(child.actual) + "]."));
-        }
-
-        child.expected = promote;
-        child.cast();
+        originallyExplicit = input.explicit;
 
-        if (promote == def.class && expected != null) {
-            actual = expected;
+        if (operation == Operation.NOT) {
+            Input childInput = new Input();
+            childInput.expected = boolean.class;
+            child.analyze(scriptRoot, scope, childInput);
+            child.cast();
+
+            output.actual = boolean.class;
+        } else if (operation == Operation.BWNOT || operation == Operation.ADD || operation == Operation.SUB) {
+            Output childOutput = child.analyze(scriptRoot, scope, new Input());
+
+            promote = AnalyzerCaster.promoteNumeric(childOutput.actual, operation != Operation.BWNOT);
+
+            if (promote == null) {
+                throw createError(new ClassCastException("cannot apply the " + operation.name + " operator " +
+                        "[" + operation.symbol + "] to the type " +
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(childOutput.actual) + "]"));
+            }
+
+            child.input.expected = promote;
+            child.cast();
+
+            if (promote == def.class && input.expected != null) {
+                output.actual = input.expected;
+            } else {
+                output.actual = promote;
+            }
         } else {
-            actual = promote;
+            throw createError(new IllegalStateException("unexpected unary operation [" + operation.name + "]"));
         }
-    }
 
-    void analyzerSub(ScriptRoot scriptRoot, Scope variables) {
-        child.analyze(scriptRoot, variables);
-
-        promote = AnalyzerCaster.promoteNumeric(child.actual, true);
-
-        if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply negative [-] to type " +
-                    "[" + PainlessLookupUtility.typeToJavaType(child.actual) + "]."));
-        }
-
-        child.expected = promote;
-        child.cast();
-
-        if (promote == def.class && expected != null) {
-            actual = expected;
-        } else {
-            actual = promote;
-        }
+        return output;
     }
 
     @Override
@@ -142,7 +97,7 @@ UnaryNode write(ClassNode classNode) {
         unaryMathNode.setChildNode(child.cast(child.write(classNode)));
 
         unaryMathNode.setLocation(location);
-        unaryMathNode.setExpressionType(actual);
+        unaryMathNode.setExpressionType(output.actual);
         unaryMathNode.setUnaryType(promote);
         unaryMathNode.setOperation(operation);
         unaryMathNode.setOriginallExplicit(originallyExplicit);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
index 840ec3b2b5114..2f1a4d6f09a72 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
@@ -42,14 +42,30 @@ public EVariable(Location location, String name) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+        AStoreable.Input storeableInput = new AStoreable.Input();
+        storeableInput.read = input.read;
+        storeableInput.expected = input.expected;
+        storeableInput.explicit = input.explicit;
+        storeableInput.internal = input.internal;
+
+        return analyze(scriptRoot, scope, storeableInput);
+    }
+
+    @Override
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
         Variable variable = scope.getVariable(location, name);
 
-        if (write && variable.isFinal()) {
+        if (input.write && variable.isFinal()) {
             throw createError(new IllegalArgumentException("Variable [" + variable.getName() + "] is read-only."));
         }
 
-        actual = variable.getType();
+        output.actual = variable.getType();
+
+        return output;
     }
 
     @Override
@@ -57,7 +73,7 @@ VariableNode write(ClassNode classNode) {
         VariableNode variableNode = new VariableNode();
 
         variableNode.setLocation(location);
-        variableNode.setExpressionType(actual);
+        variableNode.setExpressionType(output.actual);
         variableNode.setName(name);
 
         return variableNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index 31246ffe56131..684b015aa835e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -47,30 +47,47 @@ public PBrace(Location location, AExpression prefix, AExpression index) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        prefix.analyze(scriptRoot, scope);
-        prefix.expected = prefix.actual;
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+        AStoreable.Input storeableInput = new AStoreable.Input();
+        storeableInput.read = input.read;
+        storeableInput.expected = input.expected;
+        storeableInput.explicit = input.explicit;
+        storeableInput.internal = input.internal;
+
+        return analyze(scriptRoot, scope, storeableInput);
+    }
+
+    @Override
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
+        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
+        prefix.input.expected = prefixOutput.actual;
         prefix.cast();
 
-        if (prefix.actual.isArray()) {
-            sub = new PSubBrace(location, prefix.actual, index);
-        } else if (prefix.actual == def.class) {
+        if (prefixOutput.actual.isArray()) {
+            sub = new PSubBrace(location, prefixOutput.actual, index);
+        } else if (prefixOutput.actual == def.class) {
             sub = new PSubDefArray(location, index);
-        } else if (Map.class.isAssignableFrom(prefix.actual)) {
-            sub = new PSubMapShortcut(location, prefix.actual, index);
-        } else if (List.class.isAssignableFrom(prefix.actual)) {
-            sub = new PSubListShortcut(location, prefix.actual, index);
+        } else if (Map.class.isAssignableFrom(prefixOutput.actual)) {
+            sub = new PSubMapShortcut(location, prefixOutput.actual, index);
+        } else if (List.class.isAssignableFrom(prefixOutput.actual)) {
+            sub = new PSubListShortcut(location, prefixOutput.actual, index);
         } else {
             throw createError(new IllegalArgumentException("Illegal array access on type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual) + "]."));
         }
 
-        sub.write = write;
-        sub.read = read;
-        sub.expected = expected;
-        sub.explicit = explicit;
-        sub.analyze(scriptRoot, scope);
-        actual = sub.actual;
+        Input subInput = new Input();
+        subInput.write = input.write;
+        subInput.read = input.read;
+        subInput.expected = input.expected;
+        subInput.explicit = input.explicit;
+        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        output.actual = subOutput.actual;
+
+        return output;
     }
 
     @Override
@@ -81,7 +98,7 @@ BraceNode write(ClassNode classNode) {
         braceNode.setRightNode(sub.write(classNode));
 
         braceNode.setLocation(location);
-        braceNode.setExpressionType(actual);
+        braceNode.setExpressionType(output.actual);
 
         return braceNode;
     }
@@ -94,7 +111,7 @@ boolean isDefOptimized() {
     @Override
     void updateActual(Class<?> actual) {
         sub.updateActual(actual);
-        this.actual = actual;
+        this.output.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index fe4ab2658302c..4a0254ef96253 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -53,37 +53,43 @@ public PCallInvoke(Location location, AExpression prefix, String name, boolean n
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        prefix.analyze(scriptRoot, scope);
-        prefix.expected = prefix.actual;
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
+        prefix.input.expected = prefixOutput.actual;
         prefix.cast();
 
-        if (prefix.actual == def.class) {
+        if (prefixOutput.actual == def.class) {
             sub = new PSubDefCall(location, name, arguments);
         } else {
-            PainlessMethod method =
-                    scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, prefix instanceof EStatic, name, arguments.size());
+            PainlessMethod method = scriptRoot.getPainlessLookup().lookupPainlessMethod(
+                    prefixOutput.actual, prefix instanceof EStatic, name, arguments.size());
 
             if (method == null) {
                 throw createError(new IllegalArgumentException(
-                        "method [" + typeToCanonicalTypeName(prefix.actual) + ", " + name + "/" + arguments.size() + "] not found"));
+                        "method [" + typeToCanonicalTypeName(prefixOutput.actual) + ", " + name + "/" + arguments.size() + "] not found"));
             }
 
             scriptRoot.markNonDeterministic(method.annotations.containsKey(NonDeterministicAnnotation.class));
 
-            sub = new PSubCallInvoke(location, method, prefix.actual, arguments);
+            sub = new PSubCallInvoke(location, method, prefixOutput.actual, arguments);
         }
 
         if (nullSafe) {
             sub = new PSubNullSafeCallInvoke(location, sub);
         }
 
-        sub.expected = expected;
-        sub.explicit = explicit;
-        sub.analyze(scriptRoot, scope);
-        actual = sub.actual;
+        Input subInput = new Input();
+        subInput.expected = input.expected;
+        subInput.explicit = input.explicit;
+        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        output.actual = subOutput.actual;
+
+        output.statement = true;
 
-        statement = true;
+        return output;
     }
 
     @Override
@@ -94,7 +100,7 @@ CallNode write(ClassNode classNode) {
         callNode.setRightNode(sub.write(classNode));
 
         callNode.setLocation(location);
-        callNode.setExpressionType(actual);
+        callNode.setExpressionType(output.actual);
 
         return callNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index a1ad58d28570d..2380aa536edde 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -53,51 +53,66 @@ public PField(Location location, AExpression prefix, boolean nullSafe, String va
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        prefix.analyze(scriptRoot, scope);
-        prefix.expected = prefix.actual;
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+        AStoreable.Input storeableInput = new AStoreable.Input();
+        storeableInput.read = input.read;
+        storeableInput.expected = input.expected;
+        storeableInput.explicit = input.explicit;
+        storeableInput.internal = input.internal;
+
+        return analyze(scriptRoot, scope, storeableInput);
+    }
+
+    @Override
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
+        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
+        prefix.input.expected = prefixOutput.actual;
         prefix.cast();
 
-        if (prefix.actual.isArray()) {
-            sub = new PSubArrayLength(location, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), value);
-        } else if (prefix.actual == def.class) {
+        if (prefixOutput.actual.isArray()) {
+            sub = new PSubArrayLength(location, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), value);
+        } else if (prefixOutput.actual == def.class) {
             sub = new PSubDefField(location, value);
         } else {
-            PainlessField field = scriptRoot.getPainlessLookup().lookupPainlessField(prefix.actual, prefix instanceof EStatic, value);
+            PainlessField field = scriptRoot.getPainlessLookup().lookupPainlessField(prefixOutput.actual, prefix instanceof EStatic, value);
 
             if (field == null) {
                 PainlessMethod getter;
                 PainlessMethod setter;
 
-                getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
+                getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefixOutput.actual, false,
                         "get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
 
                 if (getter == null) {
-                    getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
+                    getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefixOutput.actual, false,
                             "is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
                 }
 
-                setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefix.actual, false,
+                setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(prefixOutput.actual, false,
                         "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0);
 
                 if (getter != null || setter != null) {
-                    sub = new PSubShortcut(location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter);
+                    sub = new PSubShortcut(
+                            location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), getter, setter);
                 } else {
                     EConstant index = new EConstant(location, value);
-                    index.analyze(scriptRoot, scope);
+                    index.analyze(scriptRoot, scope, new Input());
 
-                    if (Map.class.isAssignableFrom(prefix.actual)) {
-                        sub = new PSubMapShortcut(location, prefix.actual, index);
+                    if (Map.class.isAssignableFrom(prefixOutput.actual)) {
+                        sub = new PSubMapShortcut(location, prefixOutput.actual, index);
                     }
 
-                    if (List.class.isAssignableFrom(prefix.actual)) {
-                        sub = new PSubListShortcut(location, prefix.actual, index);
+                    if (List.class.isAssignableFrom(prefixOutput.actual)) {
+                        sub = new PSubListShortcut(location, prefixOutput.actual, index);
                     }
                 }
 
                 if (sub == null) {
                     throw createError(new IllegalArgumentException(
-                            "field [" + typeToCanonicalTypeName(prefix.actual) + ", " + value + "] not found"));
+                            "field [" + typeToCanonicalTypeName(prefixOutput.actual) + ", " + value + "] not found"));
                 }
             } else {
                 sub = new PSubField(location, field);
@@ -108,12 +123,15 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             sub = new PSubNullSafeField(location, sub);
         }
 
-        sub.write = write;
-        sub.read = read;
-        sub.expected = expected;
-        sub.explicit = explicit;
-        sub.analyze(scriptRoot, scope);
-        actual = sub.actual;
+        Input subInput = new Input();
+        subInput.write = input.write;
+        subInput.read = input.read;
+        subInput.expected = input.expected;
+        subInput.explicit = input.explicit;
+        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        output.actual = subOutput.actual;
+
+        return output;
     }
 
     @Override
@@ -124,7 +142,7 @@ DotNode write(ClassNode classNode) {
         dotNode.setRightNode(sub.write(classNode));
 
         dotNode.setLocation(location);
-        dotNode.setExpressionType(actual);
+        dotNode.setExpressionType(output.actual);
 
         return dotNode;
     }
@@ -137,7 +155,7 @@ boolean isDefOptimized() {
     @Override
     void updateActual(Class<?> actual) {
         sub.updateActual(actual);
-        this.actual = actual;
+        this.output.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
index bbf2a086645f7..43cdba046bd54 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
@@ -43,16 +43,21 @@ final class PSubArrayLength extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
         if ("length".equals(value)) {
-            if (write) {
+            if (input.write) {
                 throw createError(new IllegalArgumentException("Cannot write to read-only field [length] for an array."));
             }
 
-            actual = int.class;
+            output.actual = int.class;
         } else {
             throw createError(new IllegalArgumentException("Field [" + value + "] does not exist for type [" + type + "]."));
         }
+
+        return output;
     }
 
     @Override
@@ -60,7 +65,7 @@ DotSubArrayLengthNode write(ClassNode classNode) {
         DotSubArrayLengthNode dotSubArrayLengthNode = new DotSubArrayLengthNode();
 
         dotSubArrayLengthNode.setLocation(location);
-        dotSubArrayLengthNode.setExpressionType(actual);
+        dotSubArrayLengthNode.setExpressionType(output.actual);
 
         return dotSubArrayLengthNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
index 2ea986ac39625..0303ce051f7f5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
@@ -43,12 +43,18 @@ final class PSubBrace extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        index.expected = int.class;
-        index.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
+        Input indexInput = new Input();
+        indexInput.expected = int.class;
+        index.analyze(scriptRoot, scope, indexInput);
         index.cast();
 
-        actual = clazz.getComponentType();
+        output.actual = clazz.getComponentType();
+
+        return output;
     }
 
     BraceSubNode write(ClassNode classNode) {
@@ -57,7 +63,7 @@ BraceSubNode write(ClassNode classNode) {
         braceSubNode.setChildNode(index.cast(index.write(classNode)));
 
         braceSubNode.setLocation(location);
-        braceSubNode.setExpressionType(actual);
+        braceSubNode.setExpressionType(output.actual);
 
         return braceSubNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 487621b60046a..7e509682f7d47 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -47,18 +47,24 @@ final class PSubCallInvoke extends AExpression {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = method.typeParameters.get(argument);
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.expected = method.typeParameters.get(argument);
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
-        statement = true;
-        actual = method.returnType;
+        output.statement = true;
+        output.actual = method.returnType;
+
+        return output;
     }
 
     @Override
@@ -70,7 +76,7 @@ CallSubNode write(ClassNode classNode) {
         }
 
         callSubNode.setLocation(location);
-        callSubNode.setExpressionType(actual);
+        callSubNode.setExpressionType(output.actual);
         callSubNode.setMethod(method);
         callSubNode .setBox(box);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
index a39a16e26be62..060102ae8593f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
@@ -42,13 +42,18 @@ final class PSubDefArray extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        index.analyze(scriptRoot, scope);
-        index.expected = index.actual;
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
+        Output indexOutput = index.analyze(scriptRoot, scope, new Input());
+        index.input.expected = indexOutput.actual;
         index.cast();
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
-        actual = expected == null || expected == ZonedDateTime.class || explicit ? def.class : expected;
+        output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
+
+        return output;
     }
 
     @Override
@@ -58,7 +63,7 @@ BraceSubDefNode write(ClassNode classNode) {
         braceSubDefNode.setChildNode(index.cast(index.write(classNode)));
 
         braceSubDefNode.setLocation(location);
-        braceSubDefNode.setExpressionType(actual);
+        braceSubDefNode.setExpressionType(output.actual);
 
         return braceSubDefNode;
     }
@@ -70,7 +75,7 @@ boolean isDefOptimized() {
 
     @Override
     void updateActual(Class<?> actual) {
-        this.actual = actual;
+        this.output.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
index 5cface91e59b9..f810f1a634b6e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
@@ -51,23 +51,27 @@ final class PSubDefCall extends AExpression {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         parameterTypes.add(Object.class);
         int totalCaptures = 0;
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            Input expressionInput = new Input();
+            expressionInput.internal = true;
+            Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
 
-            if (expression.actual == void.class) {
+            if (expressionOutput.actual == void.class) {
                 throw createError(new IllegalArgumentException("Argument(s) cannot be of [void] type when calling method [" + name + "]."));
             }
 
-            expression.expected = expression.actual;
+            expression.input.expected = expressionOutput.actual;
             expression.cast();
-            parameterTypes.add(expression.actual);
+            parameterTypes.add(expressionOutput.actual);
 
             if (expression instanceof ILambda) {
                 ILambda lambda = (ILambda) expression;
@@ -81,7 +85,9 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
-        actual = expected == null || expected == ZonedDateTime.class || explicit ? def.class : expected;
+        output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
+
+        return output;
     }
 
     @Override
@@ -93,7 +99,7 @@ CallSubDefNode write(ClassNode classNode) {
         }
 
         callSubDefNode.setLocation(location);
-        callSubDefNode.setExpressionType(actual);
+        callSubDefNode.setExpressionType(output.actual);
         callSubDefNode.setName(name);
         callSubDefNode.setRecipe(recipe.toString());
         callSubDefNode.getPointers().addAll(pointers);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
index 317fbabd99b87..75bfa5c5755cc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
@@ -43,9 +43,14 @@ final class PSubDefField extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
-        actual = expected == null || expected == ZonedDateTime.class || explicit ? def.class : expected;
+        output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
+
+        return output;
     }
 
     @Override
@@ -53,7 +58,7 @@ DotSubDefNode write(ClassNode classNode) {
         DotSubDefNode dotSubDefNode = new DotSubDefNode();
 
         dotSubDefNode.setLocation(location);
-        dotSubDefNode.setExpressionType(actual);
+        dotSubDefNode.setExpressionType(output.actual);
         dotSubDefNode.setValue(value);
 
         return dotSubDefNode;
@@ -66,7 +71,7 @@ boolean isDefOptimized() {
 
     @Override
     void updateActual(Class<?> actual) {
-        this.actual = actual;
+        this.output.actual = actual;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index 400dbaa167c38..c9a438571bc36 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -44,13 +44,18 @@ final class PSubField extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-         if (write && Modifier.isFinal(field.javaField.getModifiers())) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
+         if (input.write && Modifier.isFinal(field.javaField.getModifiers())) {
              throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.javaField.getName() + "] " +
                      "for type [" + PainlessLookupUtility.typeToCanonicalTypeName(field.javaField.getDeclaringClass()) + "]."));
          }
 
-        actual = field.typeParameter;
+         output.actual = field.typeParameter;
+
+         return output;
     }
 
     @Override
@@ -58,7 +63,7 @@ DotSubNode write(ClassNode classNode) {
         DotSubNode dotSubNode = new DotSubNode();
 
         dotSubNode.setLocation(location);
-        dotSubNode.setExpressionType(actual);
+        dotSubNode.setExpressionType(output.actual);
         dotSubNode.setField(field);
 
         return dotSubNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index d7ef83e800e02..4dd3b46ceaa10 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -48,7 +48,10 @@ final class PSubListShortcut extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
         getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
@@ -68,15 +71,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
-        if ((read || write) && (!read || getter != null) && (!write || setter != null)) {
-            index.expected = int.class;
-            index.analyze(scriptRoot, scope);
+        if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
+            Input indexInput = new Input();
+            indexInput.expected = int.class;
+            index.analyze(scriptRoot, scope, indexInput);
             index.cast();
 
-            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
+            output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + canonicalClassName + "]."));
         }
+
+        return output;
     }
 
     @Override
@@ -86,7 +92,7 @@ ListSubShortcutNode write(ClassNode classNode) {
         listSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
 
         listSubShortcutNode.setLocation(location);
-        listSubShortcutNode.setExpressionType(actual);
+        listSubShortcutNode.setExpressionType(output.actual);
         listSubShortcutNode.setGetter(getter);
         listSubShortcutNode.setSetter(setter);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index a1468746fc038..707609e5a4947 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -48,7 +48,10 @@ final class PSubMapShortcut extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
         getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
@@ -67,15 +70,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
-        if ((read || write) && (!read || getter != null) && (!write || setter != null)) {
-            index.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
-            index.analyze(scriptRoot, scope);
+        if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
+            Input indexInput = new Input();
+            indexInput.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
+            index.analyze(scriptRoot, scope, indexInput);
             index.cast();
 
-            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
+            output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + canonicalClassName + "]."));
         }
+
+        return output;
     }
 
     @Override
@@ -85,7 +91,7 @@ MapSubShortcutNode write(ClassNode classNode) {
         mapSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
 
         mapSubShortcutNode.setLocation(location);
-        mapSubShortcutNode.setExpressionType(actual);
+        mapSubShortcutNode.setExpressionType(output.actual);
         mapSubShortcutNode.setGetter(getter);
         mapSubShortcutNode.setSetter(setter);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
index 2f5139457e1f6..83d5fded5649d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
@@ -42,12 +42,17 @@ public PSubNullSafeCallInvoke(Location location, AExpression guarded) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        guarded.analyze(scriptRoot, scope);
-        actual = guarded.actual;
-        if (actual.isPrimitive()) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        Output guardedOutput = guarded.analyze(scriptRoot, scope, new Input());
+        output.actual = guardedOutput.actual;
+        if (output.actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
+
+        return output;
     }
 
     @Override
@@ -57,7 +62,7 @@ NullSafeSubNode write(ClassNode classNode) {
         nullSafeSubNode.setChildNode(guarded.write(classNode));
 
         nullSafeSubNode.setLocation(location);
-        nullSafeSubNode.setExpressionType(actual);
+        nullSafeSubNode.setExpressionType(output.actual);
 
         return nullSafeSubNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
index 5f438644dff70..536c8b15e83c6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
@@ -37,16 +37,22 @@ public PSubNullSafeField(Location location, AStoreable guarded) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (write) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.write) {
             throw createError(new IllegalArgumentException("Can't write to null safe reference"));
         }
-        guarded.read = read;
-        guarded.analyze(scriptRoot, scope);
-        actual = guarded.actual;
-        if (actual.isPrimitive()) {
+        Input guardedInput = new Input();
+        guardedInput.read = input.read;
+        Output guardedOutput = guarded.analyze(scriptRoot, scope, guardedInput);
+        output.actual = guardedOutput.actual;
+        if (output.actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
+
+        return output;
     }
 
     @Override
@@ -66,7 +72,7 @@ NullSafeSubNode write(ClassNode classNode) {
         nullSafeSubNode.setChildNode(guarded.write(classNode));
 
         nullSafeSubNode.setLocation(location);
-        nullSafeSubNode.setExpressionType(actual);
+        nullSafeSubNode.setExpressionType(output.actual);
 
         return nullSafeSubNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index 187df63e303fe..e88fbeaac0765 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -46,7 +46,10 @@ final class PSubShortcut extends AStoreable {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        this.input = input;
+        output = new Output();
+
         if (getter != null && (getter.returnType == void.class || !getter.typeParameters.isEmpty())) {
             throw createError(new IllegalArgumentException(
                 "Illegal get shortcut on field [" + value + "] for type [" + type + "]."));
@@ -61,11 +64,13 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
-        if ((getter != null || setter != null) && (!read || getter != null) && (!write || setter != null)) {
-            actual = setter != null ? setter.typeParameters.get(0) : getter.returnType;
+        if ((getter != null || setter != null) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
+            output.actual = setter != null ? setter.typeParameters.get(0) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal shortcut on field [" + value + "] for type [" + type + "]."));
         }
+
+        return output;
     }
 
     @Override
@@ -73,7 +78,7 @@ DotSubShortcutNode write(ClassNode classNode) {
         DotSubShortcutNode dotSubShortcutNode = new DotSubShortcutNode();
 
         dotSubShortcutNode.setLocation(location);
-        dotSubShortcutNode.setExpressionType(actual);
+        dotSubShortcutNode.setExpressionType(output.actual);
         dotSubShortcutNode.setGetter(getter);
         dotSubShortcutNode.setSetter(setter);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 20ee3ec572d57..8e6db345e0b30 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -44,7 +44,10 @@ public SBlock(Location location, List<AStatement> statements) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("A block must contain at least one statement."));
         }
@@ -54,22 +57,25 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         for (AStatement statement : statements) {
             // Note that we do not need to check after the last statement because
             // there is no statement that can be unreachable after the last.
-            if (allEscape) {
+            if (output.allEscape) {
                 throw createError(new IllegalArgumentException("Unreachable statement."));
             }
 
-            statement.inLoop = inLoop;
-            statement.lastSource = lastSource && statement == last;
-            statement.lastLoop = (beginLoop || lastLoop) && statement == last;
-            statement.analyze(scriptRoot, scope);
-
-            methodEscape = statement.methodEscape;
-            loopEscape = statement.loopEscape;
-            allEscape = statement.allEscape;
-            anyContinue |= statement.anyContinue;
-            anyBreak |= statement.anyBreak;
-            statementCount += statement.statementCount;
+            Input statementInput = new Input();
+            statementInput.inLoop = input.inLoop;
+            statementInput.lastSource = input.lastSource && statement == last;
+            statementInput.lastLoop = (input.beginLoop || input.lastLoop) && statement == last;
+            Output statementOutput = statement.analyze(scriptRoot, scope, statementInput);
+
+            output.methodEscape = statementOutput.methodEscape;
+            output.loopEscape = statementOutput.loopEscape;
+            output.allEscape = statementOutput.allEscape;
+            output.anyContinue |= statementOutput.anyContinue;
+            output.anyBreak |= statementOutput.anyBreak;
+            output.statementCount += statementOutput.statementCount;
         }
+
+        return output;
     }
 
     @Override
@@ -81,8 +87,8 @@ BlockNode write(ClassNode classNode) {
         }
 
         blockNode.setLocation(location);
-        blockNode.setAllEscape(allEscape);
-        blockNode.setStatementCount(statementCount);
+        blockNode.setAllEscape(output.allEscape);
+        blockNode.setStatementCount(output.statementCount);
 
         return blockNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 8c2d2d3c03403..4b036153319ea 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -35,15 +35,20 @@ public SBreak(Location location) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!inLoop) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Break statement outside of a loop."));
         }
 
-        loopEscape = true;
-        allEscape = true;
-        anyBreak = true;
-        statementCount = 1;
+        output.loopEscape = true;
+        output.allEscape = true;
+        output.anyBreak = true;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index ed960776a19f2..f00c0b18a70fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -46,8 +46,11 @@ public SCatch(Location location, DType baseException, SDeclaration declaration,
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        declaration.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        declaration.analyze(scriptRoot, scope, new Input());
 
         Class<?> baseType = baseException.resolveType(scriptRoot.getPainlessLookup()).getType();
         Class<?> type = scope.getVariable(location, declaration.name).getType();
@@ -59,18 +62,21 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         if (block != null) {
-            block.lastSource = lastSource;
-            block.inLoop = inLoop;
-            block.lastLoop = lastLoop;
-            block.analyze(scriptRoot, scope);
-
-            methodEscape = block.methodEscape;
-            loopEscape = block.loopEscape;
-            allEscape = block.allEscape;
-            anyContinue = block.anyContinue;
-            anyBreak = block.anyBreak;
-            statementCount = block.statementCount;
+            Input blockInput = new Input();
+            blockInput.lastSource = input.lastSource;
+            blockInput.inLoop = input.inLoop;
+            blockInput.lastLoop = input.lastLoop;
+            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+
+            output.methodEscape = blockOutput.methodEscape;
+            output.loopEscape = blockOutput.loopEscape;
+            output.allEscape = blockOutput.allEscape;
+            output.anyContinue = blockOutput.anyContinue;
+            output.anyBreak = blockOutput.anyBreak;
+            output.statementCount = blockOutput.statementCount;
         }
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index fe19365095777..1f5c752f8ccef 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -35,18 +35,23 @@ public SContinue(Location location) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!inLoop) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
         }
 
-        if (lastLoop) {
+        if (input.lastLoop) {
             throw createError(new IllegalArgumentException("Extraneous continue statement."));
         }
 
-        allEscape = true;
-        anyContinue = true;
-        statementCount = 1;
+        output.allEscape = true;
+        output.anyContinue = true;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index fa85f57346908..47bbe0d123ecb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -44,12 +44,17 @@ public SDeclBlock(Location location, List<SDeclaration> declarations) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         for (SDeclaration declaration : declarations) {
-            declaration.analyze(scriptRoot, scope);
+            declaration.analyze(scriptRoot, scope, new Input());
         }
 
-        statementCount = declarations.size();
+        output.statementCount = declarations.size();
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index e379881bba154..2a28b4d029558 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DeclarationNode;
+import org.elasticsearch.painless.node.AExpression.Input;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -47,17 +48,23 @@ public SDeclaration(Location location, DType type, String name, boolean requires
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         DResolvedType resolvedType = type.resolveType(scriptRoot.getPainlessLookup());
         type = resolvedType;
 
         if (expression != null) {
-            expression.expected = resolvedType.getType();
-            expression.analyze(scriptRoot, scope);
+            AExpression.Input expressionInput = new AExpression.Input();
+            expressionInput.expected = resolvedType.getType();
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
         scope.defineVariable(location, resolvedType.getType(), name, false);
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index d95b51d8d1a0a..8d31f325a049f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -45,23 +45,28 @@ public SDo(Location location, SBlock block, AExpression condition) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         scope = scope.newLocalScope();
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
-        block.beginLoop = true;
-        block.inLoop = true;
-        block.analyze(scriptRoot, scope);
+        Input blockInput = new Input();
+        blockInput.beginLoop = true;
+        blockInput.inLoop = true;
+        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
 
-        if (block.loopEscape && !block.anyContinue) {
+        if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
-        condition.expected = boolean.class;
-        condition.analyze(scriptRoot, scope);
+        AExpression.Input conditionInput = new AExpression.Input();
+        conditionInput.expected = boolean.class;
+        condition.analyze(scriptRoot, scope, conditionInput);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -71,13 +76,15 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                 throw createError(new IllegalArgumentException("Extraneous do while loop."));
             }
 
-            if (!block.anyBreak) {
-                methodEscape = true;
-                allEscape = true;
+            if (blockOutput.anyBreak == false) {
+                output.methodEscape = true;
+                output.allEscape = true;
             }
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index a64f6451e4dd2..b5c439233b64d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -53,9 +53,12 @@ public SEach(Location location, String type, String name, AExpression expression
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        expression.analyze(scriptRoot, scope);
-        expression.expected = expression.actual;
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, new AExpression.Input());
+        expression.input.expected = expressionOutput.actual;
         expression.cast();
 
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
@@ -67,31 +70,34 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         scope = scope.newLocalScope();
         Variable variable = scope.defineVariable(location, clazz, name, true);
 
-        if (expression.actual.isArray()) {
+        if (expressionOutput.actual.isArray()) {
             sub = new SSubEachArray(location, variable, expression, block);
-        } else if (expression.actual == def.class || Iterable.class.isAssignableFrom(expression.actual)) {
+        } else if (expressionOutput.actual == def.class || Iterable.class.isAssignableFrom(expressionOutput.actual)) {
             sub = new SSubEachIterable(location, variable, expression, block);
         } else {
             throw createError(new IllegalArgumentException("Illegal for each type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expression.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expressionOutput.actual) + "]."));
         }
 
-        sub.analyze(scriptRoot, scope);
+        sub.analyze(scriptRoot, scope, input);
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous for each loop."));
         }
 
-        block.beginLoop = true;
-        block.inLoop = true;
-        block.analyze(scriptRoot, scope);
-        block.statementCount = Math.max(1, block.statementCount);
+        Input blockInput = new Input();
+        blockInput.beginLoop = true;
+        blockInput.inLoop = true;
+        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+        blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
 
-        if (block.loopEscape && !block.anyContinue) {
+        if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous for loop."));
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 2271b63ee2897..f04cefa622acd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -44,34 +44,40 @@ public SExpression(Location location, AExpression expression) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+
         Class<?> rtnType = scope.getReturnType();
         boolean isVoid = rtnType == void.class;
 
-        expression.read = lastSource && !isVoid;
-        expression.analyze(scriptRoot, scope);
+        AExpression.Input expressionInput = new AExpression.Input();
+        expressionInput.read = input.lastSource && !isVoid;
+        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
 
-        if ((lastSource == false || isVoid) && expression.statement == false) {
+        if ((input.lastSource == false || isVoid) && expressionOutput.statement == false) {
             throw createError(new IllegalArgumentException("Not a statement."));
         }
 
-        boolean rtn = lastSource && !isVoid && expression.actual != void.class;
+        boolean rtn = input.lastSource && isVoid == false && expressionOutput.actual != void.class;
 
-        expression.expected = rtn ? rtnType : expression.actual;
-        expression.internal = rtn;
+        expression.input.expected = rtn ? rtnType : expressionOutput.actual;
+        expression.input.internal = rtn;
         expression.cast();
 
-        methodEscape = rtn;
-        loopEscape = rtn;
-        allEscape = rtn;
-        statementCount = 1;
+        output = new Output();
+        output.methodEscape = rtn;
+        output.loopEscape = rtn;
+        output.allEscape = rtn;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
     StatementNode write(ClassNode classNode) {
         ExpressionNode expressionNode = expression.cast(expression.write(classNode));
 
-        if (methodEscape) {
+        if (output.methodEscape) {
             ReturnNode returnNode = new ReturnNode();
 
             returnNode.setExpressionNode(expressionNode);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index a3f71376e061c..241c50b8fe2b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -52,23 +52,26 @@ public SFor(Location location, ANode initializer, AExpression condition, AExpres
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+
         scope = scope.newLocalScope();
 
         if (initializer != null) {
             if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(scriptRoot, scope);
+                ((SDeclBlock)initializer).analyze(scriptRoot, scope, new Input());
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
-                initializer.read = false;
-                initializer.analyze(scriptRoot, scope);
+                AExpression.Input initializerInput = new AExpression.Input();
+                initializerInput.read = false;
+                AExpression.Output initializerOutput = initializer.analyze(scriptRoot, scope, initializerInput);
 
-                if (!initializer.statement) {
+                if (initializerOutput.statement == false) {
                     throw createError(new IllegalArgumentException("Not a statement."));
                 }
 
-                initializer.expected = initializer.actual;
+                initializer.input.expected = initializerOutput.actual;
                 initializer.cast();
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
@@ -76,8 +79,9 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         if (condition != null) {
-            condition.expected = boolean.class;
-            condition.analyze(scriptRoot, scope);
+            AExpression.Input conditionInput = new AExpression.Input();
+            conditionInput.expected = boolean.class;
+            condition.analyze(scriptRoot, scope, conditionInput);
             condition.cast();
 
             if (condition instanceof EBoolean) {
@@ -96,36 +100,42 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         if (afterthought != null) {
-            afterthought.read = false;
-            afterthought.analyze(scriptRoot, scope);
+            AExpression.Input afterthoughtInput = new AExpression.Input();
+            afterthoughtInput.read = false;
+            AExpression.Output afterthoughtOutput = afterthought.analyze(scriptRoot, scope, afterthoughtInput);
 
-            if (!afterthought.statement) {
+            if (afterthoughtOutput.statement == false) {
                 throw createError(new IllegalArgumentException("Not a statement."));
             }
 
-            afterthought.expected = afterthought.actual;
+            afterthought.input.expected = afterthoughtOutput.actual;
             afterthought.cast();
         }
 
+        output = new Output();
+
         if (block != null) {
-            block.beginLoop = true;
-            block.inLoop = true;
+            Input blockInput = new Input();
+            blockInput.beginLoop = true;
+            blockInput.inLoop = true;
 
-            block.analyze(scriptRoot, scope);
+            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
 
-            if (block.loopEscape && !block.anyContinue) {
+            if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous for loop."));
             }
 
-            if (continuous && !block.anyBreak) {
-                methodEscape = true;
-                allEscape = true;
+            if (continuous && blockOutput.anyBreak == false) {
+                output.methodEscape = true;
+                output.allEscape = true;
             }
 
-            block.statementCount = Math.max(1, block.statementCount);
+            blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 2ae9ef8d5d78f..96bf040099a17 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -30,6 +30,8 @@
 import org.elasticsearch.painless.ir.ReturnNode;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.node.AStatement.Input;
+import org.elasticsearch.painless.node.AStatement.Output;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.lang.invoke.MethodType;
@@ -134,9 +136,10 @@ void analyze(ScriptRoot scriptRoot) {
             throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "]."));
         }
 
-        block.lastSource = true;
-        block.analyze(scriptRoot, functionScope.newLocalScope());
-        methodEscape = block.methodEscape;
+        Input blockInput = new Input();
+        blockInput.lastSource = true;
+        Output blockOutput = block.analyze(scriptRoot, functionScope.newLocalScope(), blockInput);
+        methodEscape = blockOutput.methodEscape;
 
         if (methodEscape == false && isAutoReturnEnabled == false && returnType != void.class) {
             throw createError(new IllegalArgumentException("not all paths provide a return value " +
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index 3b95ec181dc68..a7d738670ec6f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -43,9 +43,13 @@ public SIf(Location location, AExpression condition, SBlock ifblock) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        condition.expected = boolean.class;
-        condition.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        AExpression.Input conditionInput = new AExpression.Input();
+        conditionInput.expected = boolean.class;
+        condition.analyze(scriptRoot, scope, conditionInput);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -56,15 +60,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
-        ifblock.lastSource = lastSource;
-        ifblock.inLoop = inLoop;
-        ifblock.lastLoop = lastLoop;
+        Input ifblockInput = new Input();
+        ifblockInput.lastSource = input.lastSource;
+        ifblockInput.inLoop = input.inLoop;
+        ifblockInput.lastLoop = input.lastLoop;
+
+        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
 
-        ifblock.analyze(scriptRoot, scope.newLocalScope());
+        output.anyContinue = ifblockOutput.anyContinue;
+        output.anyBreak = ifblockOutput.anyBreak;
+        output.statementCount = ifblockOutput.statementCount;
 
-        anyContinue = ifblock.anyContinue;
-        anyBreak = ifblock.anyBreak;
-        statementCount = ifblock.statementCount;
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index b3726cee6c114..f433db860ccfb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -48,9 +48,13 @@ public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        condition.expected = boolean.class;
-        condition.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        AExpression.Input conditionInput = new AExpression.Input();
+        conditionInput.expected = boolean.class;
+        condition.analyze(scriptRoot, scope, conditionInput);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -61,32 +65,36 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
-        ifblock.lastSource = lastSource;
-        ifblock.inLoop = inLoop;
-        ifblock.lastLoop = lastLoop;
+        Input ifblockInput = new Input();
+        ifblockInput.lastSource = input.lastSource;
+        ifblockInput.inLoop = input.inLoop;
+        ifblockInput.lastLoop = input.lastLoop;
 
-        ifblock.analyze(scriptRoot, scope.newLocalScope());
+        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
 
-        anyContinue = ifblock.anyContinue;
-        anyBreak = ifblock.anyBreak;
-        statementCount = ifblock.statementCount;
+        output.anyContinue = ifblockOutput.anyContinue;
+        output.anyBreak = ifblockOutput.anyBreak;
+        output.statementCount = ifblockOutput.statementCount;
 
         if (elseblock == null) {
             throw createError(new IllegalArgumentException("Extraneous else statement."));
         }
 
-        elseblock.lastSource = lastSource;
-        elseblock.inLoop = inLoop;
-        elseblock.lastLoop = lastLoop;
+        Input elseblockInput = new Input();
+        elseblockInput.lastSource = input.lastSource;
+        elseblockInput.inLoop = input.inLoop;
+        elseblockInput.lastLoop = input.lastLoop;
+
+        Output elseblockOutput = elseblock.analyze(scriptRoot, scope.newLocalScope(), elseblockInput);
 
-        elseblock.analyze(scriptRoot, scope.newLocalScope());
+        output.methodEscape = ifblockOutput.methodEscape && elseblockOutput.methodEscape;
+        output.loopEscape = ifblockOutput.loopEscape && elseblockOutput.loopEscape;
+        output.allEscape = ifblockOutput.allEscape && elseblockOutput.allEscape;
+        output.anyContinue |= elseblockOutput.anyContinue;
+        output.anyBreak |= elseblockOutput.anyBreak;
+        output.statementCount = Math.max(ifblockOutput.statementCount, elseblockOutput.statementCount);
 
-        methodEscape = ifblock.methodEscape && elseblock.methodEscape;
-        loopEscape = ifblock.loopEscape && elseblock.loopEscape;
-        allEscape = ifblock.allEscape && elseblock.allEscape;
-        anyContinue |= elseblock.anyContinue;
-        anyBreak |= elseblock.anyBreak;
-        statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index e1d181d78025b..23a5d4183edbc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -40,7 +40,10 @@ public SReturn(Location location, AExpression expression) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         if (expression == null) {
             if (scope.getReturnType() != void.class) {
                 throw location.createError(new ClassCastException("Cannot cast from " +
@@ -48,17 +51,20 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(void.class) + "]."));
             }
         } else {
-            expression.expected = scope.getReturnType();
-            expression.internal = true;
-            expression.analyze(scriptRoot, scope);
+            AExpression.Input expressionInput = new AExpression.Input();
+            expressionInput.expected = scope.getReturnType();
+            expressionInput.internal = true;
+            expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
-        methodEscape = true;
-        loopEscape = true;
-        allEscape = true;
+        output.methodEscape = true;
+        output.loopEscape = true;
+        output.allEscape = true;
+
+        output.statementCount = 1;
 
-        statementCount = 1;
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index e7c954828eb07..5e1846457d515 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -52,13 +52,18 @@ final class SSubEachArray extends AStatement {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
-        array = scope.defineInternalVariable(location, expression.actual, "array" + location.getOffset(), true);
+        array = scope.defineInternalVariable(location, expression.output.actual, "array" + location.getOffset(), true);
         index = scope.defineInternalVariable(location, int.class, "index" + location.getOffset(), true);
-        indexed = expression.actual.getComponentType();
+        indexed = expression.output.actual.getComponentType();
         cast = AnalyzerCaster.getLegalCast(location, indexed, variable.getType(), true, true);
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 72d29b9f03ddc..4198452ff8e79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -57,23 +57,28 @@ final class SSubEachIterable extends AStatement {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
         iterator = scope.defineInternalVariable(location, Iterator.class, "itr" + location.getOffset(), true);
 
-        if (expression.actual == def.class) {
+        if (expression.output.actual == def.class) {
             method = null;
         } else {
-            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expression.actual, false, "iterator", 0);
+            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expression.output.actual, false, "iterator", 0);
 
             if (method == null) {
                     throw createError(new IllegalArgumentException(
-                            "method [" + typeToCanonicalTypeName(expression.actual) + ", iterator/0] not found"));
+                            "method [" + typeToCanonicalTypeName(expression.output.actual) + ", iterator/0] not found"));
             }
         }
 
         cast = AnalyzerCaster.getLegalCast(location, def.class, variable.getType(), true, true);
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index e3132f024f52b..86b40fafa378b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -41,15 +41,21 @@ public SThrow(Location location, AExpression expression) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        expression.expected = Exception.class;
-        expression.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        AExpression.Input expressionInput = new AExpression.Input();
+        expressionInput.expected = Exception.class;
+        expression.analyze(scriptRoot, scope, expressionInput);
         expression.cast();
 
-        methodEscape = true;
-        loopEscape = true;
-        allEscape = true;
-        statementCount = 1;
+        output.methodEscape = true;
+        output.loopEscape = true;
+        output.allEscape = true;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 6c73b625a64ac..7798fd13b13a2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -46,42 +46,49 @@ public STry(Location location, SBlock block, List<SCatch> catches) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous try statement."));
         }
 
-        block.lastSource = lastSource;
-        block.inLoop = inLoop;
-        block.lastLoop = lastLoop;
+        Input blockInput = new Input();
+        blockInput.lastSource = input.lastSource;
+        blockInput.inLoop = input.inLoop;
+        blockInput.lastLoop = input.lastLoop;
 
-        block.analyze(scriptRoot, scope.newLocalScope());
+        Output blockOutput = block.analyze(scriptRoot, scope.newLocalScope(), blockInput);
 
-        methodEscape = block.methodEscape;
-        loopEscape = block.loopEscape;
-        allEscape = block.allEscape;
-        anyContinue = block.anyContinue;
-        anyBreak = block.anyBreak;
+        output.methodEscape = blockOutput.methodEscape;
+        output.loopEscape = blockOutput.loopEscape;
+        output.allEscape = blockOutput.allEscape;
+        output.anyContinue = blockOutput.anyContinue;
+        output.anyBreak = blockOutput.anyBreak;
 
         int statementCount = 0;
 
         for (SCatch catc : catches) {
-            catc.lastSource = lastSource;
-            catc.inLoop = inLoop;
-            catc.lastLoop = lastLoop;
+            Input catchInput = new Input();
+            catchInput.lastSource = input.lastSource;
+            catchInput.inLoop = input.inLoop;
+            catchInput.lastLoop = input.lastLoop;
 
-            catc.analyze(scriptRoot, scope.newLocalScope());
+            Output catchOutput = catc.analyze(scriptRoot, scope.newLocalScope(), catchInput);
 
-            methodEscape &= catc.methodEscape;
-            loopEscape &= catc.loopEscape;
-            allEscape &= catc.allEscape;
-            anyContinue |= catc.anyContinue;
-            anyBreak |= catc.anyBreak;
+            output.methodEscape &= catchOutput.methodEscape;
+            output.loopEscape &= catchOutput.loopEscape;
+            output.allEscape &= catchOutput.allEscape;
+            output.anyContinue |= catchOutput.anyContinue;
+            output.anyBreak |= catchOutput.anyBreak;
 
-            statementCount = Math.max(statementCount, catc.statementCount);
+            statementCount = Math.max(statementCount, catchOutput.statementCount);
         }
 
-        this.statementCount = block.statementCount + statementCount;
+        output.statementCount = blockOutput.statementCount + statementCount;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 8213e5e1bc559..eead51786e2d3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -45,11 +45,15 @@ public SWhile(Location location, AExpression condition, SBlock block) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         scope = scope.newLocalScope();
 
-        condition.expected = boolean.class;
-        condition.analyze(scriptRoot, scope);
+        AExpression.Input conditionInput = new AExpression.Input();
+        conditionInput.expected = boolean.class;
+        condition.analyze(scriptRoot, scope, conditionInput);
         condition.cast();
 
         if (condition instanceof EBoolean) {
@@ -65,24 +69,27 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         if (block != null) {
-            block.beginLoop = true;
-            block.inLoop = true;
+            Input blockInput = new Input();
+            blockInput.beginLoop = true;
+            blockInput.inLoop = true;
 
-            block.analyze(scriptRoot, scope);
+            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
 
-            if (block.loopEscape && !block.anyContinue) {
+            if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous while loop."));
             }
 
-            if (continuous && !block.anyBreak) {
-                methodEscape = true;
-                allEscape = true;
+            if (continuous && blockOutput.anyBreak == false) {
+                output.methodEscape = true;
+                output.allEscape = true;
             }
 
-            block.statementCount = Math.max(1, block.statementCount);
+            blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
index adde835deb104..6fb5b44d957aa 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
@@ -28,7 +28,6 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.BlendedTermQuery;
-import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.IntervalQuery;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
@@ -1122,33 +1121,33 @@ public void testIntervalQueries() {
         assertTermsEqual(result.extractions, new Term("field", "term1"), new Term("field", "term2"),
             new Term("field", "term3"), new Term("field", "term4"));
 
-        source = Intervals.ordered(Intervals.term("term1"), XIntervals.wildcard(new BytesRef("a*")));
+        source = Intervals.ordered(Intervals.term("term1"), Intervals.wildcard(new BytesRef("a*")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertThat(result.verified, is(false));
         assertThat(result.matchAllDocs, is(false));
         assertThat(result.minimumShouldMatch, equalTo(1));
         assertTermsEqual(result.extractions, new Term("field", "term1"));
 
-        source = Intervals.ordered(XIntervals.wildcard(new BytesRef("a*")));
+        source = Intervals.ordered(Intervals.wildcard(new BytesRef("a*")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
-        source = Intervals.or(Intervals.term("b"), XIntervals.wildcard(new BytesRef("a*")));
+        source = Intervals.or(Intervals.term("b"), Intervals.wildcard(new BytesRef("a*")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
-        source = Intervals.ordered(Intervals.term("term1"), XIntervals.prefix(new BytesRef("a")));
+        source = Intervals.ordered(Intervals.term("term1"), Intervals.prefix(new BytesRef("a")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertThat(result.verified, is(false));
         assertThat(result.matchAllDocs, is(false));
         assertThat(result.minimumShouldMatch, equalTo(1));
         assertTermsEqual(result.extractions, new Term("field", "term1"));
 
-        source = Intervals.ordered(XIntervals.prefix(new BytesRef("a")));
+        source = Intervals.ordered(Intervals.prefix(new BytesRef("a")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
-        source = Intervals.or(Intervals.term("b"), XIntervals.prefix(new BytesRef("a")));
+        source = Intervals.or(Intervals.term("b"), Intervals.prefix(new BytesRef("a")));
         result = analyze(new IntervalQuery("field", source), Version.CURRENT);
         assertEquals(Result.UNKNOWN, result);
 
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..f639c17a35f5a
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+b6f880fa08a44fcb2d50808f9eeb6189a293ce27
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index d90402c25e434..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0748be5811dfe6725847d2e87890a990c58cc3de
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..5c1e0b2f6fa43
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+9c5b8619795f69c225b5ec37b87cb34de0feccd4
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index a5003b5fb51a2..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f693cd60ad8ca9b7d3082f7b9ee6054b9c819b48
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..656f63c68d194
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+421e13b9fe09523e094ac708204d62d4ea5b6618
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index d82cbaef39b0b..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-72c34e18af81ee1d18e9927fb95690fe056cbd4f
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..951d97032f9fb
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+ff4ae9f3f3b0bc497f98c9bc47e943525669fc99
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 2b08ae87a25d9..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-655438348dcad9a98b5affa76caa3d67aa4bee51
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..19846e43636c3
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+dd6430c037566cd3852b73b2ec31e59de24cfe58
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 9a051c8320ae1..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b99147dad649fce0b0423e41f90c79e0f2fba2b7
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..147bbb3192d24
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+dd4ca22b151a98a21e255bc1c54f0fadfee5ca4d
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 92d6378f04c08..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-00ce3e23cf7aba8c1b3e777de92fd31ec1d4d814
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..86cee0ecdd039
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+e85f94d2747ddb560af0bc4d15f0cde45cf3ff30
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 427fa3cc39049..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2b03f79d61517d8e6a8744dbd89e61ad661f6a62
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
deleted file mode 100644
index 5203969bcf5c0..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
deleted file mode 100644
index 5203969bcf5c0..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java
index c473e4e010552..f7a621c4125a9 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.SegmentCommitInfo;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.index.mapper.ParsedDocument;
 
 import java.io.IOException;
@@ -89,7 +90,7 @@ public synchronized MergePolicy.OneMerge getNextMerge() {
                         StreamSupport.stream(e.getLastCommittedSegmentInfos().spliterator(), false).collect(Collectors.toList());
                 segmentsReference.set(segments);
                 // trigger a background merge that will be managed by the concurrent merge scheduler
-                e.forceMerge(randomBoolean(), 0, false, false, false);
+                e.forceMerge(randomBoolean(), 0, false, false, false, UUIDs.randomBase64UUID());
                 /*
                  * Merging happens in the background on a merge thread, and the maybeDie handler is invoked on yet another thread; we have
                  * to wait for these events to finish.
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
index 0e5236f9b1171..f8a31c5ec9214 100644
--- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
@@ -80,9 +80,9 @@
       cluster.remote_info: {}
 
   - match: { test_remote_cluster.connected: true }
-  - match: { test_remote_cluster.address: $remote_ip }
-  - gt:    { test_remote_cluster.num_sockets_connected: 0}
-  - match: { test_remote_cluster.max_socket_connections: 10}
+  - match: { test_remote_cluster.proxy_address: $remote_ip }
+  - gt:    { test_remote_cluster.num_proxy_sockets_connected: 0}
+  - match: { test_remote_cluster.max_proxy_socket_connections: 10}
   - match: { test_remote_cluster.initial_connect_timeout: "30s" }
   - match: { test_remote_cluster.mode: "proxy" }
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
index d37273cfd41bf..6ee594b099c43 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
@@ -53,6 +53,18 @@
         "type":"boolean",
         "description":"Verbose mode. Display column headers",
         "default":false
+      },
+      "expand_wildcards":{
+        "type":"enum",
+        "options":[
+          "open",
+          "closed",
+          "hidden",
+          "none",
+          "all"
+        ],
+        "default": ["all"],
+        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
index 76c749cc7d3bd..d4a7eb3b051d2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
@@ -107,6 +107,18 @@
         "type":"boolean",
         "description":"If set to true segment stats will include stats for segments that are not currently loaded into memory",
         "default":false
+      },
+      "expand_wildcards":{
+        "type":"enum",
+        "options":[
+          "open",
+          "closed",
+          "hidden",
+          "none",
+          "all"
+        ],
+        "default": "all",
+        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
index 7911a8e244218..2a21ff3725b6a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
@@ -33,6 +33,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
index d6f0e9ababc6d..017705082d189 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
@@ -97,6 +97,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
index 4084d3be608a4..93a450afff88a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -47,6 +47,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index 93f6d63bbc2d2..9be09b49cc54c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -70,6 +70,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
index eed6f8bf082a2..d56c3313a0bf0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
@@ -47,6 +47,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
index b1b1d8214c47d..64c10a520c7c4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
@@ -53,6 +53,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
index 539088199d683..f26c8e77a06a6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
index d77d47f0d739b..53fdf44bb36a1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
index d6bdcef95815f..7539f44a81eed 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
@@ -39,6 +39,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
index a19786a7938f9..66e5ce92cbbe5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
@@ -51,6 +51,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
index 2a09d8bdfa679..c854d0e8fd841 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
@@ -39,6 +39,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
index 1eada8f831db6..35138b920466f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
@@ -51,6 +51,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
index 5eaf62840d841..6036b75bb83e4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
@@ -45,6 +45,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
index ccacf91d93095..90a1274ecb059 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
@@ -39,6 +39,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
index 800ea8d30d68d..e238c4fc38afc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
@@ -69,10 +69,11 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
-        "default":"all",
+        "default": ["all"],
         "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       },
       "local":{
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
index 4c5191f2f320a..0e71b6d395777 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
@@ -55,6 +55,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
index 17274e8bbd78a..5a27f6d06b83c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
@@ -41,6 +41,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
index c19fa3bf5262f..4a1dea974f750 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
@@ -73,6 +73,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
index 862fdde4ac395..68cfdf25aeea2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
@@ -49,6 +49,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
index 14050d626df83..1dab468ce4ff4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
index c95f5ea3b48f3..18a42ec11a45e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
@@ -44,6 +44,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
index 75bf3a340c311..66fe23bab8ba2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
@@ -53,6 +53,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
index 7eee4eeec5fd0..950e0a62489fa 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
index b99b19799dab9..83430a9a85600 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
@@ -41,6 +41,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
index b397963f99d2f..7e48e99916171 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
@@ -51,6 +51,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
index 0940313154a74..0a8960f2f9e89 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
@@ -137,6 +137,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
index 93628fdd106d0..406fbacda307c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
@@ -45,6 +45,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
index 5ff6f073b77ec..3becec003a9e6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
@@ -69,6 +69,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
index c87b850ab148f..eadf240192394 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 2411c309c7dff..1e0c232efa055 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -89,6 +89,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
index 70ee592520be9..74b7055b4c4b0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
@@ -55,6 +55,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
index f3286cd932c4e..1d239376b3fe8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -47,6 +47,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
index 591371798d440..299c4f705d292 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
@@ -70,6 +70,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
new file mode 100644
index 0000000000000..6866ff595fefd
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
@@ -0,0 +1,150 @@
+---
+"Test cat aliases output with a hidden index with a hidden alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            test_alias:
+              is_hidden: true
+
+  - do:
+      cat.aliases: {}
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+  - do:
+      cat.aliases:
+        name: test_alias
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+
+  - do:
+      cat.aliases:
+        expand_wildcards: ["open","closed"]
+
+  - match:
+      $body: |
+        /^
+        $/
+---
+"Test cat aliases output with a hidden index with a visible alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            test_alias: {}
+  - do:
+      cat.aliases: {}
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+  - do:
+      cat.aliases:
+        name: test_alias
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+---
+"Test cat aliases output with a visible index with a hidden alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+          aliases:
+            test_alias:
+              is_hidden: true
+  - do:
+      cat.aliases: {}
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+  - do:
+      cat.aliases:
+        name: test_alias
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
new file mode 100644
index 0000000000000..3a4fe28c85996
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
@@ -0,0 +1,243 @@
+---
+"Test cat indices output for hidden index":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+  - do:
+      cat.indices: {}
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        expand_wildcards: ["all"]
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+---
+"Test cat indices output for dot-hidden index and dot-prefixed pattern":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+  - do:
+      indices.create:
+        index: .index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+  - do:
+      cat.indices: {}
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        index: ".*"
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           \.index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+---
+"Test cat indices output with a hidden index with a visible alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            alias1: {}
+  - do:
+      cat.indices:
+        index: "i*"
+        # Can't use a bare wildcard here because Security replaces wildcards
+        # it with all matching authorized indices/aliases, including the visible
+        # alias
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        expand_wildcards: ["open", "hidden"]
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+  - do:
+      cat.indices:
+        index: alias1
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+---
+"Test cat indices output with a hidden index with a hidden alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            alias1:
+              is_hidden: true
+  - do:
+      cat.indices: {}
+
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        expand_wildcards: ["all"]
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+  - do:
+      cat.indices:
+        index: alias1
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+---
+"Test cat indices output with a hidden index, dot-hidden alias and dot pattern":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            .alias1:
+              is_hidden: true
+  - do:
+      cat.indices: {}
+  - match:
+      $body: |
+        /^$/
+  - do:
+      cat.indices:
+        index: ".*"
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
index 49afbb8afc714..be3d6c60a3fee 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
@@ -8,6 +8,8 @@ setup:
                 properties:
                   date:
                     type: date
+                  date_nanos:
+                    type: date_nanos
                   keyword:
                     type: keyword
                   long:
@@ -850,3 +852,138 @@ setup:
   - length: { aggregations.test.buckets: 1 }
   - match: { aggregations.test.buckets.0.key.date: "2017-10-21T00:00:00.000-02:00" }
   - match: { aggregations.test.buckets.0.doc_count: 2 }
+
+---
+"date_histogram on date_nanos":
+  - skip:
+      version: " - 7.6.99"
+      reason: Fixed in 7.7.0
+  - do:
+      index:
+        index:   test
+        id:      7
+        body:    { "date_nanos": "2017-11-21T01:00:00" }
+        refresh: true
+  - do:
+      index:
+        index:   test
+        id:      8
+        body:    { "date_nanos": "2017-11-22T01:00:00" }
+        refresh: true
+  - do:
+      index:
+        index:   test
+        id:      9
+        body:    { "date_nanos": "2017-11-22T02:00:00" }
+        refresh: true
+  - do:
+      search:
+        index: test
+        body:
+          aggregations:
+            test:
+              composite:
+                sources:
+                  - date:
+                      date_histogram:
+                        field: date_nanos
+                        calendar_interval: 1d
+                        format: iso8601 # Format makes the comparisons a little more obvious
+              aggregations:
+                avg:
+                  avg:
+                    field: date_nanos
+
+  - match: { hits.total.value: 9 }
+  - match: { hits.total.relation: eq }
+  - length: { aggregations.test.buckets: 2 }
+  - match: { aggregations.test.buckets.0.key.date: "2017-11-21T00:00:00.000Z" }
+  - match: { aggregations.test.buckets.0.doc_count: 1 }
+  - match: { aggregations.test.buckets.0.avg.value_as_string: "2017-11-21T01:00:00.000Z" }
+  - match: { aggregations.test.buckets.1.key.date: "2017-11-22T00:00:00.000Z" }
+  - match: { aggregations.test.buckets.1.doc_count: 2 }
+  - match: { aggregations.test.buckets.1.avg.value_as_string: "2017-11-22T01:30:00.000Z" }
+
+---
+"Terms source from sorted":
+  - do:
+        indices.create:
+          index: sorted_test
+          body:
+              settings:
+                sort.field: keyword
+              mappings:
+                properties:
+                  keyword:
+                    type: keyword
+                  long:
+                    type: long
+
+
+  - do:
+      index:
+        index:   sorted_test
+        id:      2
+        refresh: true
+        body:  { "keyword": "foo", "long": 1 }
+
+  - do:
+      search:
+        index: sorted_test
+        body:
+          aggregations:
+            test:
+              composite:
+                sources:
+                  - keyword:
+                      terms:
+                        field: keyword
+
+  - match: {hits.total.value: 1}
+  - length: { aggregations.test.buckets: 1 }
+  - match: { aggregations.test.buckets.0.key.keyword: "foo" }
+  - match: { aggregations.test.buckets.0.doc_count: 1 }
+
+---
+"Terms source from part of sorted":
+  - skip:
+      version: " - 7.6.99"
+      reason:  fixed in 7.7.0.
+
+  - do:
+        indices.create:
+          index: sorted_test
+          body:
+              settings:
+                sort.field: [keyword, long]
+              mappings:
+                properties:
+                  keyword:
+                    type: keyword
+                  long:
+                    type: long
+
+
+  - do:
+      index:
+        index:   sorted_test
+        id:      2
+        refresh: true
+        body:  { "keyword": "foo", "long": 1 }
+
+  - do:
+      search:
+        index: sorted_test
+        body:
+          aggregations:
+            test:
+              composite:
+                sources:
+                  - keyword:
+                      terms:
+                        field: keyword
+
+  - match: {hits.total.value: 1}
+  - length: { aggregations.test.buckets: 1 }
+  - match: { aggregations.test.buckets.0.key.keyword: "foo" }
+  - match: { aggregations.test.buckets.0.doc_count: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml
new file mode 100644
index 0000000000000..bd7bb925a7adb
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml
@@ -0,0 +1,98 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_replicas: 0
+          mappings:
+            properties:
+              field1:
+                type: long
+              field2:
+                type: long
+              field3:
+                type: long
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test
+          - field1: 100
+          - index:
+              _index: test
+          - field1: 200
+          - index:
+              _index: test
+          - field1: 300
+            field2: 300
+
+---
+"match all":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          aggs:
+            missing_agg:
+              missing:
+                field: field3
+
+  - match: { hits.total: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { aggregations.missing_agg.doc_count: 3 }
+
+---
+"match some":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          aggs:
+            missing_agg:
+              missing:
+                field: field2
+
+  - match: { hits.total: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { aggregations.missing_agg.doc_count: 2 }
+
+---
+"match none":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          aggs:
+            missing_agg:
+              missing:
+                field: field1
+
+  - match: { hits.total: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { aggregations.missing_agg.doc_count: 0 }
+
+---
+"missing param":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          aggs:
+            missing_agg:
+              missing:
+                field: field3
+                missing: 1
+
+  - match: { hits.total: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { aggregations.missing_agg.doc_count: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
new file mode 100644
index 0000000000000..16442d2b3d823
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
@@ -0,0 +1,64 @@
+---
+setup:
+  - do:
+      indices.create:
+          index: test-index
+          body:
+            mappings:
+              "properties":
+                "k1":
+                  "type": "keyword"
+                "k2":
+                  "type": "keyword"
+                  "ignore_above": 3
+  - do:
+      bulk:
+        index: test-index
+        refresh: true
+        body:
+          - '{"index": {"_id": "1"}}'
+          - '{"k1": "123", "k2" : "123"}'
+          - '{"index": {"_id": "2"}}'
+          - '{"k1": "1234", "k2" : "1234"}'
+
+---
+"Plain Highligher should skip highlighting ignored keyword values":
+  - skip:
+      version: " - 7.9.99"
+      reason: "skip highlighting of ignored values was introduced in 7.7"
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            prefix:
+              k1: "12"
+          highlight:
+            require_field_match: false
+            fields:
+              k2:
+                type: plain
+
+  - match: {hits.hits.0.highlight.k2.0: "<em>123</em>"}
+  - is_false: hits.hits.1.highlight # no highlight for a value that was ignored
+
+---
+"Unified Highligher should skip highlighting ignored keyword values":
+  - skip:
+      version: " - 7.9.99"
+      reason: "skip highlighting of ignored values was introduced in 7.7"
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            prefix:
+              k1: "12"
+          highlight:
+            require_field_match: false
+            fields:
+              k2:
+                type: unified
+
+  - match: {hits.hits.0.highlight.k2.0: "<em>123</em>"}
+  - is_false: hits.hits.1.highlight # no highlight for a value that was ignored
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
index e0b6827aa7995..7135c8642736c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
@@ -55,94 +55,7 @@ teardown:
       cluster.get_settings:
         flat_settings: true
 
-  - match: {search.allow_expensive_queries: null}
-
-  ### Prefix
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            prefix:
-              text:
-                value: out
-
-  - match: { hits.total.value: 3 }
-
-  ### Fuzzy
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            fuzzy:
-              text:
-                value: outwide
-
-  - match: { hits.total.value: 3 }
-
-
-  ### Regexp
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            regexp:
-              text:
-                value: .*ou.*id.*
-
-  - match: { hits.total.value: 3 }
-
-  ### Wildcard
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            wildcard:
-              text:
-                value: out?ide
-
-  - match: { hits.total.value: 3 }
-
-  ### Range on text
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text:
-                gte: "theres"
-
-  - match: { hits.total.value: 2 }
-
-  ### Range on keyword
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text.raw:
-                gte : "Outside it is cold and wet"
-
-  - match: { hits.total.value: 2 }
-
-  ### Nested
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            nested:
-              path: "nested1"
-              query:
-                bool:
-                  must: [{"match": {"nested1.foo": "bar2"}}]
-
-  - match: { hits.total.value: 1 }
+  - is_false: search.allow_expensive_queries
 
   ### Update setting to false
   - do:
@@ -232,99 +145,3 @@ teardown:
               query:
                 bool:
                   must: [{"match" : {"nested1.foo" : "bar2"}}]
-
-  ### Revert setting to true
-  - do:
-      cluster.put_settings:
-        body:
-          transient:
-            search.allow_expensive_queries: "true"
-        flat_settings: true
-
-  - match: {transient: {search.allow_expensive_queries: "true"}}
-
-  ### Prefix
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            prefix:
-              text:
-                value: out
-
-  - match: { hits.total.value: 3 }
-
-  ### Fuzzy
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            fuzzy:
-              text:
-                value: outwide
-
-  - match: { hits.total.value: 3 }
-
-  ### Regexp
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            regexp:
-              text:
-                value: .*ou.*id.*
-
-  - match: { hits.total.value: 3 }
-
-  ### Wildcard
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            wildcard:
-              text:
-                value: out?ide
-
-  - match: { hits.total.value: 3 }
-
-  ### Range on text
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text:
-                gte: "theres"
-
-  - match: { hits.total.value: 2 }
-
-  ### Range on keyword
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text.raw:
-                gte: "Outside it is cold and wet"
-
-  - match: { hits.total.value: 2 }
-
-  ### Nested
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            nested:
-              path: "nested1"
-              query:
-                bool:
-                  must: [{"match": {"nested1.foo": "bar2"}}]
-
-  - match: { hits.total.value: 1 }
diff --git a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..242bbfc9bd604
--- /dev/null
+++ b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+0365c37a03123ee8e30f75e44a1cb7d5ddd2fc52
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 54cf1bac15e42..0000000000000
--- a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1963afb27f340df8fc304d377971424832f4ce1a
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..056fe4a00cbb5
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+d56b30f75b2df92da8c6c0965ce72e7abb86347b
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 3297020fb5caa..0000000000000
--- a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fdff4122e8b8a2dbbc9de24be6963e7d7e33b794
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..fe30b9975cab4
--- /dev/null
+++ b/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 3fd9e819ce181..0000000000000
--- a/server/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ca406661129d35008411365d2b6e747dc39378af
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..c3ef488e826f8
--- /dev/null
+++ b/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+2e56cc12d2f77d82946299b66f3416f9e621b2f3
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 11adaa175d1af..0000000000000
--- a/server/licenses/lucene-grouping-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-db053d5861406393254c28f6e46767879b504bb3
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..cfa7fa961780b
--- /dev/null
+++ b/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+5cddb5b65e7ead641483dcc2ffb0e50ad8d26eb7
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 99a2bbe095ea3..0000000000000
--- a/server/licenses/lucene-highlighter-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f5520ee7145f5d1ef02c7dc87483255d81b5bc6c
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..94a18fd3c3913
--- /dev/null
+++ b/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+cfabaedd80fe600cc7fda5ee12d90927fa96d87c
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index e05ece509f26e..0000000000000
--- a/server/licenses/lucene-join-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2d81c0a3473cc865e7c4858890b7fbfb869bfbf8
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..ae7da5c259f69
--- /dev/null
+++ b/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+e90fbcc53531978fc03ef847ba396d4cdd89c7e4
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 2d8258caac8ab..0000000000000
--- a/server/licenses/lucene-memory-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6d009afeb485307dce111afb8bb157ebbbb0f212
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..8f6c2fe6d63c1
--- /dev/null
+++ b/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+dd0b4cef132a50b3fa919f214a5316fcc78c46ea
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 0b228e2a8fa12..0000000000000
--- a/server/licenses/lucene-misc-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3a62908ec9eb6e826a56e697322c4c6b6c9a8573
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..4b2a346309f3f
--- /dev/null
+++ b/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+c90cc35089afc3f7802668c3969b5e7391b6d15a
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index c159ab983010d..0000000000000
--- a/server/licenses/lucene-queries-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d71d54ed6e0cf482ce16cf4f419441d83f646827
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..3b5ec91e4f938
--- /dev/null
+++ b/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+e324233cb8f069e4f6abcbab47368a83c3696f36
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 10be987e9c169..0000000000000
--- a/server/licenses/lucene-queryparser-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-181915a7d21b73dff16591b20cdee22648e4181f
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..58d7518e7aefc
--- /dev/null
+++ b/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+985a451e5f564c84271419a446e044ab589d6f22
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index fb143f9c9c3bf..0000000000000
--- a/server/licenses/lucene-sandbox-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d3d0bb76d9f4a5368d286a934615dbca7703b3d8
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..f51b876ad5fdf
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+beff7cafe0fa5330b9b915825b69321faf0fcaa9
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index fd23c6f7e0389..0000000000000
--- a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-95b9fd35e91a34c090ecf301d4dc29cabd198e6f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..736c206ba1b48
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+b9256d3a2a64d79435a4c726af8a3c28c2b77d7f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index d4fc63b96d40f..0000000000000
--- a/server/licenses/lucene-spatial3d-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1c8da46c3a172830372dfc23e18e9151bb14562c
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..586728768c5b4
--- /dev/null
+++ b/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+f38949db273a910e94a57229db2d8f3e4aef5e1f
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1 b/server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 51bc432029f38..0000000000000
--- a/server/licenses/lucene-suggest-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b3ad5d3476ed85a529892962d057518555ccfcc9
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/lucene/queries/XIntervals.java b/server/src/main/java/org/apache/lucene/queries/XIntervals.java
deleted file mode 100644
index 2de278e3037ce..0000000000000
--- a/server/src/main/java/org/apache/lucene/queries/XIntervals.java
+++ /dev/null
@@ -1,861 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lucene.queries;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.queries.intervals.IntervalIterator;
-import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
-import org.apache.lucene.queries.intervals.IntervalQuery;
-import org.apache.lucene.queries.intervals.Intervals;
-import org.apache.lucene.queries.intervals.IntervalsSource;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.MatchesIterator;
-import org.apache.lucene.search.MatchesUtils;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.QueryVisitor;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * Replacement for {@link Intervals#wildcard(BytesRef)} and {@link Intervals#prefix(BytesRef)}
- * until LUCENE-9050 is merged
- */
-public final class XIntervals {
-
-    private XIntervals() {}
-
-    public static IntervalsSource wildcard(BytesRef wildcard) {
-        CompiledAutomaton ca = new CompiledAutomaton(WildcardQuery.toAutomaton(new Term("", wildcard)));
-        return new MultiTermIntervalsSource(ca, 128, wildcard.utf8ToString());
-    }
-
-    public static IntervalsSource prefix(BytesRef prefix) {
-        CompiledAutomaton ca = new CompiledAutomaton(PrefixQuery.toAutomaton(prefix));
-        return new MultiTermIntervalsSource(ca, 128, prefix.utf8ToString());
-    }
-
-    public static IntervalsSource multiterm(CompiledAutomaton ca, String label) {
-        return new MultiTermIntervalsSource(ca, 128, label);
-    }
-
-    static class MultiTermIntervalsSource extends IntervalsSource {
-
-        private final CompiledAutomaton automaton;
-        private final int maxExpansions;
-        private final String pattern;
-
-        MultiTermIntervalsSource(CompiledAutomaton automaton, int maxExpansions, String pattern) {
-            this.automaton = automaton;
-            if (maxExpansions > BooleanQuery.getMaxClauseCount()) {
-                throw new IllegalArgumentException("maxExpansions [" + maxExpansions
-                    + "] cannot be greater than BooleanQuery.getMaxClauseCount [" + BooleanQuery.getMaxClauseCount() + "]");
-            }
-            this.maxExpansions = maxExpansions;
-            this.pattern = pattern;
-        }
-
-        @Override
-        public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
-            Terms terms = ctx.reader().terms(field);
-            if (terms == null) {
-                return null;
-            }
-            List<IntervalIterator> subSources = new ArrayList<>();
-            TermsEnum te = automaton.getTermsEnum(terms);
-            BytesRef term;
-            int count = 0;
-            while ((term = te.next()) != null) {
-                subSources.add(TermIntervalsSource.intervals(term, te));
-                if (++count > maxExpansions) {
-                    throw new IllegalStateException("Automaton [" + this.pattern + "] expanded to too many terms (limit "
-                        + maxExpansions + ")");
-                }
-            }
-            if (subSources.size() == 0) {
-                return null;
-            }
-            return new DisjunctionIntervalIterator(subSources);
-        }
-
-        @Override
-        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
-            Terms terms = ctx.reader().terms(field);
-            if (terms == null) {
-                return null;
-            }
-            List<MatchesIterator> subMatches = new ArrayList<>();
-            TermsEnum te = automaton.getTermsEnum(terms);
-            BytesRef term;
-            int count = 0;
-            while ((term = te.next()) != null) {
-                MatchesIterator mi = XIntervals.TermIntervalsSource.matches(te, doc);
-                if (mi != null) {
-                    subMatches.add(mi);
-                    if (count++ > maxExpansions) {
-                        throw new IllegalStateException("Automaton " + term + " expanded to too many terms (limit " + maxExpansions + ")");
-                    }
-                }
-            }
-            MatchesIterator mi = MatchesUtils.disjunction(subMatches);
-            if (mi == null) {
-                return null;
-            }
-            return new IntervalMatchesIterator() {
-                @Override
-                public int gaps() {
-                    return 0;
-                }
-
-                @Override
-                public int width() {
-                    return 1;
-                }
-
-                @Override
-                public boolean next() throws IOException {
-                    return mi.next();
-                }
-
-                @Override
-                public int startPosition() {
-                    return mi.startPosition();
-                }
-
-                @Override
-                public int endPosition() {
-                    return mi.endPosition();
-                }
-
-                @Override
-                public int startOffset() throws IOException {
-                    return mi.startOffset();
-                }
-
-                @Override
-                public int endOffset() throws IOException {
-                    return mi.endOffset();
-                }
-
-                @Override
-                public MatchesIterator getSubMatches() throws IOException {
-                    return mi.getSubMatches();
-                }
-
-                @Override
-                public Query getQuery() {
-                    return mi.getQuery();
-                }
-            };
-        }
-
-        @Override
-        public void visit(String field, QueryVisitor visitor) {
-            visitor.visitLeaf(new IntervalQuery(field, this));
-        }
-
-        @Override
-        public int minExtent() {
-            return 1;
-        }
-
-        @Override
-        public Collection<IntervalsSource> pullUpDisjunctions() {
-            return Collections.singleton(this);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            MultiTermIntervalsSource that = (MultiTermIntervalsSource) o;
-            return maxExpansions == that.maxExpansions &&
-                Objects.equals(automaton, that.automaton) &&
-                Objects.equals(pattern, that.pattern);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(automaton, maxExpansions, pattern);
-        }
-
-        @Override
-        public String toString() {
-            return "MultiTerm(" + pattern + ")";
-        }
-    }
-
-    static class DisiWrapper {
-
-        public final DocIdSetIterator iterator;
-        public final IntervalIterator intervals;
-        public final long cost;
-        public final float matchCost; // the match cost for two-phase iterators, 0 otherwise
-        public int doc; // the current doc, used for comparison
-        public DisiWrapper next; // reference to a next element, see #topList
-
-        // An approximation of the iterator, or the iterator itself if it does not
-        // support two-phase iteration
-        public final DocIdSetIterator approximation;
-
-        DisiWrapper(IntervalIterator iterator) {
-            this.intervals = iterator;
-            this.iterator = iterator;
-            this.cost = iterator.cost();
-            this.doc = -1;
-            this.approximation = iterator;
-            this.matchCost = iterator.matchCost();
-        }
-
-    }
-
-    static final class DisiPriorityQueue implements Iterable<DisiWrapper> {
-
-        static int leftNode(int node) {
-            return ((node + 1) << 1) - 1;
-        }
-
-        static int rightNode(int leftNode) {
-            return leftNode + 1;
-        }
-
-        static int parentNode(int node) {
-            return ((node + 1) >>> 1) - 1;
-        }
-
-        private final DisiWrapper[] heap;
-        private int size;
-
-        DisiPriorityQueue(int maxSize) {
-            heap = new DisiWrapper[maxSize];
-            size = 0;
-        }
-
-        public int size() {
-            return size;
-        }
-
-        public DisiWrapper top() {
-            return heap[0];
-        }
-
-        /** Get the list of scorers which are on the current doc. */
-        DisiWrapper topList() {
-            final DisiWrapper[] heap = this.heap;
-            final int size = this.size;
-            DisiWrapper list = heap[0];
-            list.next = null;
-            if (size >= 3) {
-                list = topList(list, heap, size, 1);
-                list = topList(list, heap, size, 2);
-            } else if (size == 2 && heap[1].doc == list.doc) {
-                list = prepend(heap[1], list);
-            }
-            return list;
-        }
-
-        // prepend w1 (iterator) to w2 (list)
-        private DisiWrapper prepend(DisiWrapper w1, DisiWrapper w2) {
-            w1.next = w2;
-            return w1;
-        }
-
-        private DisiWrapper topList(DisiWrapper list, DisiWrapper[] heap,
-                                    int size, int i) {
-            final DisiWrapper w = heap[i];
-            if (w.doc == list.doc) {
-                list = prepend(w, list);
-                final int left = leftNode(i);
-                final int right = left + 1;
-                if (right < size) {
-                    list = topList(list, heap, size, left);
-                    list = topList(list, heap, size, right);
-                } else if (left < size && heap[left].doc == list.doc) {
-                    list = prepend(heap[left], list);
-                }
-            }
-            return list;
-        }
-
-        public DisiWrapper add(DisiWrapper entry) {
-            final DisiWrapper[] heap = this.heap;
-            final int size = this.size;
-            heap[size] = entry;
-            upHeap(size);
-            this.size = size + 1;
-            return heap[0];
-        }
-
-        public DisiWrapper pop() {
-            final DisiWrapper[] heap = this.heap;
-            final DisiWrapper result = heap[0];
-            final int i = --size;
-            heap[0] = heap[i];
-            heap[i] = null;
-            downHeap(i);
-            return result;
-        }
-
-        DisiWrapper updateTop() {
-            downHeap(size);
-            return heap[0];
-        }
-
-        void upHeap(int i) {
-            final DisiWrapper node = heap[i];
-            final int nodeDoc = node.doc;
-            int j = parentNode(i);
-            while (j >= 0 && nodeDoc < heap[j].doc) {
-                heap[i] = heap[j];
-                i = j;
-                j = parentNode(j);
-            }
-            heap[i] = node;
-        }
-
-        void downHeap(int size) {
-            int i = 0;
-            final DisiWrapper node = heap[0];
-            int j = leftNode(i);
-            if (j < size) {
-                int k = rightNode(j);
-                if (k < size && heap[k].doc < heap[j].doc) {
-                    j = k;
-                }
-                if (heap[j].doc < node.doc) {
-                    do {
-                        heap[i] = heap[j];
-                        i = j;
-                        j = leftNode(i);
-                        k = rightNode(j);
-                        if (k < size && heap[k].doc < heap[j].doc) {
-                            j = k;
-                        }
-                    } while (j < size && heap[j].doc < node.doc);
-                    heap[i] = node;
-                }
-            }
-        }
-
-        @Override
-        public Iterator<DisiWrapper> iterator() {
-            return Arrays.asList(heap).subList(0, size).iterator();
-        }
-
-    }
-
-    static class DisjunctionDISIApproximation extends DocIdSetIterator {
-
-        final DisiPriorityQueue subIterators;
-        final long cost;
-
-        DisjunctionDISIApproximation(DisiPriorityQueue subIterators) {
-            this.subIterators = subIterators;
-            long cost = 0;
-            for (DisiWrapper w : subIterators) {
-                cost += w.cost;
-            }
-            this.cost = cost;
-        }
-
-        @Override
-        public long cost() {
-            return cost;
-        }
-
-        @Override
-        public int docID() {
-            return subIterators.top().doc;
-        }
-
-        @Override
-        public int nextDoc() throws IOException {
-            DisiWrapper top = subIterators.top();
-            final int doc = top.doc;
-            do {
-                top.doc = top.approximation.nextDoc();
-                top = subIterators.updateTop();
-            } while (top.doc == doc);
-
-            return top.doc;
-        }
-
-        @Override
-        public int advance(int target) throws IOException {
-            DisiWrapper top = subIterators.top();
-            do {
-                top.doc = top.approximation.advance(target);
-                top = subIterators.updateTop();
-            } while (top.doc < target);
-
-            return top.doc;
-        }
-    }
-
-    static class DisjunctionIntervalIterator extends IntervalIterator {
-
-        final DocIdSetIterator approximation;
-        final PriorityQueue<IntervalIterator> intervalQueue;
-        final DisiPriorityQueue disiQueue;
-        final List<IntervalIterator> iterators;
-        final float matchCost;
-
-        IntervalIterator current = EMPTY;
-
-        DisjunctionIntervalIterator(List<IntervalIterator> iterators) {
-            this.disiQueue = new DisiPriorityQueue(iterators.size());
-            for (IntervalIterator it : iterators) {
-                disiQueue.add(new DisiWrapper(it));
-            }
-            this.approximation = new DisjunctionDISIApproximation(disiQueue);
-            this.iterators = iterators;
-            this.intervalQueue = new PriorityQueue<>(iterators.size()) {
-                @Override
-                protected boolean lessThan(IntervalIterator a, IntervalIterator b) {
-                    return a.end() < b.end() || (a.end() == b.end() && a.start() >= b.start());
-                }
-            };
-            float costsum = 0;
-            for (IntervalIterator it : iterators) {
-                costsum += it.cost();
-            }
-            this.matchCost = costsum;
-        }
-
-        @Override
-        public float matchCost() {
-            return matchCost;
-        }
-
-        @Override
-        public int start() {
-            return current.start();
-        }
-
-        @Override
-        public int end() {
-            return current.end();
-        }
-
-        @Override
-        public int gaps() {
-            return current.gaps();
-        }
-
-        private void reset() throws IOException {
-            intervalQueue.clear();
-            for (DisiWrapper dw = disiQueue.topList(); dw != null; dw = dw.next) {
-                dw.intervals.nextInterval();
-                intervalQueue.add(dw.intervals);
-            }
-            current = EMPTY;
-        }
-
-        @Override
-        public int nextInterval() throws IOException {
-            if (current == EMPTY || current == EXHAUSTED) {
-                if (intervalQueue.size() > 0) {
-                    current = intervalQueue.top();
-                }
-                return current.start();
-            }
-            int start = current.start(), end = current.end();
-            while (intervalQueue.size() > 0 && contains(intervalQueue.top(), start, end)) {
-                IntervalIterator it = intervalQueue.pop();
-                if (it != null && it.nextInterval() != NO_MORE_INTERVALS) {
-                    intervalQueue.add(it);
-                }
-            }
-            if (intervalQueue.size() == 0) {
-                current = EXHAUSTED;
-                return NO_MORE_INTERVALS;
-            }
-            current = intervalQueue.top();
-            return current.start();
-        }
-
-        private boolean contains(IntervalIterator it, int start, int end) {
-            return start >= it.start() && start <= it.end() && end >= it.start() && end <= it.end();
-        }
-
-        @Override
-        public int docID() {
-            return approximation.docID();
-        }
-
-        @Override
-        public int nextDoc() throws IOException {
-            int doc = approximation.nextDoc();
-            reset();
-            return doc;
-        }
-
-        @Override
-        public int advance(int target) throws IOException {
-            int doc = approximation.advance(target);
-            reset();
-            return doc;
-        }
-
-        @Override
-        public long cost() {
-            return approximation.cost();
-        }
-    }
-
-    private static final IntervalIterator EMPTY = new IntervalIterator() {
-
-        @Override
-        public int docID() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int nextDoc() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int advance(int target) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public long cost() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int start() {
-            return -1;
-        }
-
-        @Override
-        public int end() {
-            return -1;
-        }
-
-        @Override
-        public int gaps() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int nextInterval() {
-            return NO_MORE_INTERVALS;
-        }
-
-        @Override
-        public float matchCost() {
-            return 0;
-        }
-    };
-
-    private static final IntervalIterator EXHAUSTED = new IntervalIterator() {
-
-        @Override
-        public int docID() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int nextDoc() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int advance(int target) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public long cost() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int start() {
-            return NO_MORE_INTERVALS;
-        }
-
-        @Override
-        public int end() {
-            return NO_MORE_INTERVALS;
-        }
-
-        @Override
-        public int gaps() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int nextInterval() {
-            return NO_MORE_INTERVALS;
-        }
-
-        @Override
-        public float matchCost() {
-            return 0;
-        }
-    };
-
-    static class TermIntervalsSource extends IntervalsSource {
-
-        final BytesRef term;
-
-        TermIntervalsSource(BytesRef term) {
-            this.term = term;
-        }
-
-        @Override
-        public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
-            Terms terms = ctx.reader().terms(field);
-            if (terms == null)
-                return null;
-            if (terms.hasPositions() == false) {
-                throw new IllegalArgumentException("Cannot create an IntervalIterator over field " + field
-                    + " because it has no indexed positions");
-            }
-            TermsEnum te = terms.iterator();
-            if (te.seekExact(term) == false) {
-                return null;
-            }
-            return intervals(term, te);
-        }
-
-        static IntervalIterator intervals(BytesRef term, TermsEnum te) throws IOException {
-            PostingsEnum pe = te.postings(null, PostingsEnum.POSITIONS);
-            float cost = termPositionsCost(te);
-            return new IntervalIterator() {
-
-                @Override
-                public int docID() {
-                    return pe.docID();
-                }
-
-                @Override
-                public int nextDoc() throws IOException {
-                    int doc = pe.nextDoc();
-                    reset();
-                    return doc;
-                }
-
-                @Override
-                public int advance(int target) throws IOException {
-                    int doc = pe.advance(target);
-                    reset();
-                    return doc;
-                }
-
-                @Override
-                public long cost() {
-                    return pe.cost();
-                }
-
-                int pos = -1, upto;
-
-                @Override
-                public int start() {
-                    return pos;
-                }
-
-                @Override
-                public int end() {
-                    return pos;
-                }
-
-                @Override
-                public int gaps() {
-                    return 0;
-                }
-
-                @Override
-                public int nextInterval() throws IOException {
-                    if (upto <= 0)
-                        return pos = NO_MORE_INTERVALS;
-                    upto--;
-                    return pos = pe.nextPosition();
-                }
-
-                @Override
-                public float matchCost() {
-                    return cost;
-                }
-
-                private void reset() throws IOException {
-                    if (pe.docID() == NO_MORE_DOCS) {
-                        upto = -1;
-                        pos = NO_MORE_INTERVALS;
-                    }
-                    else {
-                        upto = pe.freq();
-                        pos = -1;
-                    }
-                }
-
-                @Override
-                public String toString() {
-                    return term.utf8ToString() + ":" + super.toString();
-                }
-            };
-        }
-
-        @Override
-        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
-            Terms terms = ctx.reader().terms(field);
-            if (terms == null)
-                return null;
-            if (terms.hasPositions() == false) {
-                throw new IllegalArgumentException("Cannot create an IntervalIterator over field " + field
-                    + " because it has no indexed positions");
-            }
-            TermsEnum te = terms.iterator();
-            if (te.seekExact(term) == false) {
-                return null;
-            }
-            return matches(te, doc);
-        }
-
-        static IntervalMatchesIterator matches(TermsEnum te, int doc) throws IOException {
-            PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS);
-            if (pe.advance(doc) != doc) {
-                return null;
-            }
-            return new IntervalMatchesIterator() {
-
-                @Override
-                public int gaps() {
-                    return 0;
-                }
-
-                @Override
-                public int width() {
-                    return 1;
-                }
-
-                int upto = pe.freq();
-                int pos = -1;
-
-                @Override
-                public boolean next() throws IOException {
-                    if (upto <= 0) {
-                        pos = IntervalIterator.NO_MORE_INTERVALS;
-                        return false;
-                    }
-                    upto--;
-                    pos = pe.nextPosition();
-                    return true;
-                }
-
-                @Override
-                public int startPosition() {
-                    return pos;
-                }
-
-                @Override
-                public int endPosition() {
-                    return pos;
-                }
-
-                @Override
-                public int startOffset() throws IOException {
-                    return pe.startOffset();
-                }
-
-                @Override
-                public int endOffset() throws IOException {
-                    return pe.endOffset();
-                }
-
-                @Override
-                public MatchesIterator getSubMatches() {
-                    return null;
-                }
-
-                @Override
-                public Query getQuery() {
-                    throw new UnsupportedOperationException();
-                }
-            };
-        }
-
-        @Override
-        public int minExtent() {
-            return 1;
-        }
-
-        @Override
-        public Collection<IntervalsSource> pullUpDisjunctions() {
-            return Collections.singleton(this);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(term);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            TermIntervalsSource that = (TermIntervalsSource) o;
-            return Objects.equals(term, that.term);
-        }
-
-        @Override
-        public String toString() {
-            return term.utf8ToString();
-        }
-
-        @Override
-        public void visit(String field, QueryVisitor visitor) {
-            visitor.consumeTerms(new IntervalQuery(field, this), new Term(field, term));
-        }
-
-        private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
-
-        private static final int TERM_OPS_PER_POS = 7;
-
-        static float termPositionsCost(TermsEnum termsEnum) throws IOException {
-            int docFreq = termsEnum.docFreq();
-            assert docFreq > 0;
-            long totalTermFreq = termsEnum.totalTermFreq();
-            float expOccurrencesInMatchingDoc = totalTermFreq / (float) docFreq;
-            return TERM_POSNS_SEEK_OPS_PER_DOC + expOccurrencesInMatchingDoc * TERM_OPS_PER_POS;
-        }
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
index 4ea78a17fd0e9..6f33f75692294 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
@@ -286,7 +286,7 @@ enum Metrics {
         OS("os"),
         PROCESS("process"),
         JVM("jvm"),
-        THREAD_POOL("threadPool"),
+        THREAD_POOL("thread_pool"),
         TRANSPORT("transport"),
         HTTP("http"),
         PLUGINS("plugins"),
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index 205404a564aea..4524a2c082c65 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -19,12 +19,17 @@
 
 package org.elasticsearch.action.admin.cluster.node.stats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * A request to get node (cluster) level stats.
@@ -32,18 +37,7 @@
 public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
 
     private CommonStatsFlags indices = new CommonStatsFlags();
-    private boolean os;
-    private boolean process;
-    private boolean jvm;
-    private boolean threadPool;
-    private boolean fs;
-    private boolean transport;
-    private boolean http;
-    private boolean breaker;
-    private boolean script;
-    private boolean discovery;
-    private boolean ingest;
-    private boolean adaptiveSelection;
+    private final Set<String> requestedMetrics = new HashSet<>();
 
     public NodesStatsRequest() {
         super((String[]) null);
@@ -51,19 +45,25 @@ public NodesStatsRequest() {
 
     public NodesStatsRequest(StreamInput in) throws IOException {
         super(in);
+
         indices = new CommonStatsFlags(in);
-        os = in.readBoolean();
-        process = in.readBoolean();
-        jvm = in.readBoolean();
-        threadPool = in.readBoolean();
-        fs = in.readBoolean();
-        transport = in.readBoolean();
-        http = in.readBoolean();
-        breaker = in.readBoolean();
-        script = in.readBoolean();
-        discovery = in.readBoolean();
-        ingest = in.readBoolean();
-        adaptiveSelection = in.readBoolean();
+        requestedMetrics.clear();
+        if (in.getVersion().before(Version.V_7_7_0)) {
+            addOrRemoveMetric(in.readBoolean(), Metric.OS.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.PROCESS.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.JVM.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.THREAD_POOL.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.FS.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.TRANSPORT.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.HTTP.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.BREAKER.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.SCRIPT.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.DISCOVERY.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.INGEST.metricName());
+            addOrRemoveMetric(in.readBoolean(), Metric.ADAPTIVE_SELECTION.metricName());
+        } else {
+            requestedMetrics.addAll(in.readStringList());
+        }
     }
 
     /**
@@ -79,18 +79,7 @@ public NodesStatsRequest(String... nodesIds) {
      */
     public NodesStatsRequest all() {
         this.indices.all();
-        this.os = true;
-        this.process = true;
-        this.jvm = true;
-        this.threadPool = true;
-        this.fs = true;
-        this.transport = true;
-        this.http = true;
-        this.breaker = true;
-        this.script = true;
-        this.discovery = true;
-        this.ingest = true;
-        this.adaptiveSelection = true;
+        this.requestedMetrics.addAll(Metric.allMetrics());
         return this;
     }
 
@@ -99,18 +88,7 @@ public NodesStatsRequest all() {
      */
     public NodesStatsRequest clear() {
         this.indices.clear();
-        this.os = false;
-        this.process = false;
-        this.jvm = false;
-        this.threadPool = false;
-        this.fs = false;
-        this.transport = false;
-        this.http = false;
-        this.breaker = false;
-        this.script = false;
-        this.discovery = false;
-        this.ingest = false;
-        this.adaptiveSelection = false;
+        this.requestedMetrics.clear();
         return this;
     }
 
@@ -139,14 +117,14 @@ public NodesStatsRequest indices(boolean indices) {
      * Should the node OS be returned.
      */
     public boolean os() {
-        return this.os;
+        return Metric.OS.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node OS be returned.
      */
     public NodesStatsRequest os(boolean os) {
-        this.os = os;
+        addOrRemoveMetric(os, Metric.OS.metricName());
         return this;
     }
 
@@ -154,14 +132,14 @@ public NodesStatsRequest os(boolean os) {
      * Should the node Process be returned.
      */
     public boolean process() {
-        return this.process;
+        return Metric.PROCESS.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node Process be returned.
      */
     public NodesStatsRequest process(boolean process) {
-        this.process = process;
+        addOrRemoveMetric(process, Metric.PROCESS.metricName());
         return this;
     }
 
@@ -169,14 +147,14 @@ public NodesStatsRequest process(boolean process) {
      * Should the node JVM be returned.
      */
     public boolean jvm() {
-        return this.jvm;
+        return Metric.JVM.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node JVM be returned.
      */
     public NodesStatsRequest jvm(boolean jvm) {
-        this.jvm = jvm;
+        addOrRemoveMetric(jvm, Metric.JVM.metricName());
         return this;
     }
 
@@ -184,14 +162,14 @@ public NodesStatsRequest jvm(boolean jvm) {
      * Should the node Thread Pool be returned.
      */
     public boolean threadPool() {
-        return this.threadPool;
+        return Metric.THREAD_POOL.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node Thread Pool be returned.
      */
     public NodesStatsRequest threadPool(boolean threadPool) {
-        this.threadPool = threadPool;
+        addOrRemoveMetric(threadPool, Metric.THREAD_POOL.metricName());
         return this;
     }
 
@@ -199,14 +177,14 @@ public NodesStatsRequest threadPool(boolean threadPool) {
      * Should the node file system stats be returned.
      */
     public boolean fs() {
-        return this.fs;
+        return Metric.FS.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node file system stats be returned.
      */
     public NodesStatsRequest fs(boolean fs) {
-        this.fs = fs;
+        addOrRemoveMetric(fs, Metric.FS.metricName());
         return this;
     }
 
@@ -214,14 +192,14 @@ public NodesStatsRequest fs(boolean fs) {
      * Should the node Transport be returned.
      */
     public boolean transport() {
-        return this.transport;
+        return Metric.TRANSPORT.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node Transport be returned.
      */
     public NodesStatsRequest transport(boolean transport) {
-        this.transport = transport;
+        addOrRemoveMetric(transport, Metric.TRANSPORT.metricName());
         return this;
     }
 
@@ -229,90 +207,144 @@ public NodesStatsRequest transport(boolean transport) {
      * Should the node HTTP be returned.
      */
     public boolean http() {
-        return this.http;
+        return Metric.HTTP.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node HTTP be returned.
      */
     public NodesStatsRequest http(boolean http) {
-        this.http = http;
+        addOrRemoveMetric(http, Metric.HTTP.metricName());
         return this;
     }
 
     public boolean breaker() {
-        return this.breaker;
+        return Metric.BREAKER.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node's circuit breaker stats be returned.
      */
     public NodesStatsRequest breaker(boolean breaker) {
-        this.breaker = breaker;
+        addOrRemoveMetric(breaker, Metric.BREAKER.metricName());
         return this;
     }
 
     public boolean script() {
-        return script;
+        return Metric.SCRIPT.containedIn(requestedMetrics);
     }
 
     public NodesStatsRequest script(boolean script) {
-        this.script = script;
+        addOrRemoveMetric(script, Metric.SCRIPT.metricName());
         return this;
     }
 
 
     public boolean discovery() {
-        return this.discovery;
+        return Metric.DISCOVERY.containedIn(requestedMetrics);
     }
 
     /**
      * Should the node's discovery stats be returned.
      */
     public NodesStatsRequest discovery(boolean discovery) {
-        this.discovery = discovery;
+        addOrRemoveMetric(discovery, Metric.DISCOVERY.metricName());
         return this;
     }
 
     public boolean ingest() {
-        return ingest;
+        return Metric.INGEST.containedIn(requestedMetrics);
     }
 
     /**
      * Should ingest statistics be returned.
      */
     public NodesStatsRequest ingest(boolean ingest) {
-        this.ingest = ingest;
+        addOrRemoveMetric(ingest, Metric.INGEST.metricName());
         return this;
     }
 
     public boolean adaptiveSelection() {
-        return adaptiveSelection;
+        return Metric.ADAPTIVE_SELECTION.containedIn(requestedMetrics);
     }
 
     /**
      * Should adaptiveSelection statistics be returned.
      */
     public NodesStatsRequest adaptiveSelection(boolean adaptiveSelection) {
-        this.adaptiveSelection = adaptiveSelection;
+        addOrRemoveMetric(adaptiveSelection, Metric.ADAPTIVE_SELECTION.metricName());
         return this;
     }
 
+    /**
+     * Helper method for adding and removing metrics.
+     * @param includeMetric Whether or not to include a metric.
+     * @param metricName Name of the metric to include or remove.
+     */
+    private void addOrRemoveMetric(boolean includeMetric, String metricName) {
+        if (includeMetric) {
+            requestedMetrics.add(metricName);
+        } else {
+            requestedMetrics.remove(metricName);
+        }
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         indices.writeTo(out);
-        out.writeBoolean(os);
-        out.writeBoolean(process);
-        out.writeBoolean(jvm);
-        out.writeBoolean(threadPool);
-        out.writeBoolean(fs);
-        out.writeBoolean(transport);
-        out.writeBoolean(http);
-        out.writeBoolean(breaker);
-        out.writeBoolean(script);
-        out.writeBoolean(discovery);
-        out.writeBoolean(ingest);
-        out.writeBoolean(adaptiveSelection);
+        if (out.getVersion().before(Version.V_7_7_0)) {
+            out.writeBoolean(Metric.OS.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.PROCESS.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.JVM.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.THREAD_POOL.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.FS.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.TRANSPORT.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.HTTP.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.BREAKER.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.SCRIPT.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.DISCOVERY.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.INGEST.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.ADAPTIVE_SELECTION.containedIn(requestedMetrics));
+        } else {
+            out.writeStringArray(requestedMetrics.toArray(String[]::new));
+        }
+    }
+
+    /**
+     * An enumeration of the "core" sections of metrics that may be requested
+     * from the nodes stats endpoint. Eventually this list will be pluggable.
+     */
+    private enum Metric {
+        OS("os"),
+        PROCESS("process"),
+        JVM("jvm"),
+        THREAD_POOL("thread_pool"),
+        FS("fs"),
+        TRANSPORT("transport"),
+        HTTP("http"),
+        BREAKER("breaker"),
+        SCRIPT("script"),
+        DISCOVERY("discovery"),
+        INGEST("ingest"),
+        ADAPTIVE_SELECTION("adaptiveSelection");
+
+        private String metricName;
+
+        Metric(String name) {
+            this.metricName = name;
+        }
+
+        String metricName() {
+            return this.metricName;
+        }
+
+        boolean containedIn(Set<String> metricNames) {
+            return metricNames.contains(this.metricName());
+        }
+
+        static Set<String> allMetrics() {
+            return Arrays.stream(values()).map(Metric::metricName).collect(Collectors.toSet());
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 0d2cac9b401dd..5984c7276add7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.cluster.snapshots.restore;
 
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -40,9 +41,9 @@
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 
 /**
@@ -60,7 +61,6 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private boolean includeGlobalState = false;
     private boolean partial = false;
     private boolean includeAliases = true;
-    private Settings settings = EMPTY_SETTINGS;
     private Settings indexSettings = EMPTY_SETTINGS;
     private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
 
@@ -90,7 +90,9 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException {
         includeGlobalState = in.readBoolean();
         partial = in.readBoolean();
         includeAliases = in.readBoolean();
-        settings = readSettingsFromStream(in);
+        if (in.getVersion().before(Version.V_7_7_0)) {
+            readSettingsFromStream(in); // formerly the unused settings field
+        }
         indexSettings = readSettingsFromStream(in);
         ignoreIndexSettings = in.readStringArray();
     }
@@ -108,7 +110,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(includeGlobalState);
         out.writeBoolean(partial);
         out.writeBoolean(includeAliases);
-        writeSettingsToStream(settings, out);
+        if (out.getVersion().before(Version.V_7_7_0)) {
+            writeSettingsToStream(Settings.EMPTY, out); // formerly the unused settings field
+        }
         writeSettingsToStream(indexSettings, out);
         out.writeStringArray(ignoreIndexSettings);
     }
@@ -128,9 +132,6 @@ public ActionRequestValidationException validate() {
         if (indicesOptions == null) {
             validationException = addValidationError("indicesOptions is missing", validationException);
         }
-        if (settings == null) {
-            validationException = addValidationError("settings are missing", validationException);
-        }
         if (indexSettings == null) {
             validationException = addValidationError("indexSettings are missing", validationException);
         }
@@ -324,74 +325,6 @@ public RestoreSnapshotRequest partial(boolean partial) {
         return this;
     }
 
-    /**
-     * Sets repository-specific restore settings.
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param settings repository-specific snapshot settings
-     * @return this request
-     */
-    public RestoreSnapshotRequest settings(Settings settings) {
-        this.settings = settings;
-        return this;
-    }
-
-    /**
-     * Sets repository-specific restore settings.
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param settings repository-specific snapshot settings
-     * @return this request
-     */
-    public RestoreSnapshotRequest settings(Settings.Builder settings) {
-        this.settings = settings.build();
-        return this;
-    }
-
-    /**
-     * Sets repository-specific restore settings in JSON or YAML format
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param source repository-specific snapshot settings
-     * @param xContentType the content type of the source
-     * @return this request
-     */
-    public RestoreSnapshotRequest settings(String source, XContentType xContentType) {
-        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
-        return this;
-    }
-
-    /**
-     * Sets repository-specific restore settings
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param source repository-specific snapshot settings
-     * @return this request
-     */
-    public RestoreSnapshotRequest settings(Map<String, Object> source) {
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-            builder.map(source);
-            settings(Strings.toString(builder), builder.contentType());
-        } catch (IOException e) {
-            throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
-        }
-        return this;
-    }
-
-    /**
-     * Returns repository-specific restore settings
-     *
-     * @return restore settings
-     */
-    public Settings settings() {
-        return this.settings;
-    }
-
     /**
      * Sets the list of index settings and index settings groups that shouldn't be restored from snapshot
      */
@@ -522,11 +455,6 @@ public RestoreSnapshotRequest source(Map<String, Object> source) {
                 }
             } else if (name.equals("partial")) {
                 partial(nodeBooleanValue(entry.getValue(), "partial"));
-            } else if (name.equals("settings")) {
-                if (!(entry.getValue() instanceof Map)) {
-                    throw new IllegalArgumentException("malformed settings section");
-                }
-                settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("include_global_state")) {
                 includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state");
             } else if (name.equals("include_aliases")) {
@@ -586,13 +514,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("include_global_state", includeGlobalState);
         builder.field("partial", partial);
         builder.field("include_aliases", includeAliases);
-        if (settings != null) {
-            builder.startObject("settings");
-            if (settings.isEmpty() == false) {
-                settings.toXContent(builder, params);
-            }
-            builder.endObject();
-        }
         if (indexSettings != null) {
             builder.startObject("index_settings");
             if (indexSettings.isEmpty() == false) {
@@ -629,7 +550,6 @@ public boolean equals(Object o) {
             Objects.equals(indicesOptions, that.indicesOptions) &&
             Objects.equals(renamePattern, that.renamePattern) &&
             Objects.equals(renameReplacement, that.renameReplacement) &&
-            Objects.equals(settings, that.settings) &&
             Objects.equals(indexSettings, that.indexSettings) &&
             Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings);
     }
@@ -637,7 +557,7 @@ public boolean equals(Object o) {
     @Override
     public int hashCode() {
         int result = Objects.hash(snapshot, repository, indicesOptions, renamePattern, renameReplacement, waitForCompletion,
-            includeGlobalState, partial, includeAliases, settings, indexSettings);
+            includeGlobalState, partial, includeAliases, indexSettings);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(ignoreIndexSettings);
         return result;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
index f530261644776..08d509ce3ac74 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
@@ -127,60 +127,6 @@ public RestoreSnapshotRequestBuilder setRenameReplacement(String renameReplaceme
         return this;
     }
 
-
-    /**
-     * Sets repository-specific restore settings.
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param settings repository-specific snapshot settings
-     * @return this builder
-     */
-    public RestoreSnapshotRequestBuilder setSettings(Settings settings) {
-        request.settings(settings);
-        return this;
-    }
-
-    /**
-     * Sets repository-specific restore settings.
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param settings repository-specific snapshot settings
-     * @return this builder
-     */
-    public RestoreSnapshotRequestBuilder setSettings(Settings.Builder settings) {
-        request.settings(settings);
-        return this;
-    }
-
-    /**
-     * Sets repository-specific restore settings in JSON or YAML format
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param source repository-specific snapshot settings
-     * @param xContentType the content type of the source
-     * @return this builder
-     */
-    public RestoreSnapshotRequestBuilder setSettings(String source, XContentType xContentType) {
-        request.settings(source, xContentType);
-        return this;
-    }
-
-    /**
-     * Sets repository-specific restore settings
-     * <p>
-     * See repository documentation for more information.
-     *
-     * @param source repository-specific snapshot settings
-     * @return this builder
-     */
-    public RestoreSnapshotRequestBuilder setSettings(Map<String, Object> source) {
-        request.settings(source);
-        return this;
-    }
-
     /**
      * If this parameter is set to true the operation will wait for completion of restore process before returning.
      *
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
index c596449c0e6b6..a6299cbee1ab8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
@@ -19,8 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.forcemerge;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -53,6 +56,15 @@ public static final class Defaults {
     private boolean onlyExpungeDeletes = Defaults.ONLY_EXPUNGE_DELETES;
     private boolean flush = Defaults.FLUSH;
 
+    private static final Version FORCE_MERGE_UUID_VERSION = Version.V_7_7_0;
+
+    /**
+     * Force merge UUID to store in the live commit data of a shard under
+     * {@link org.elasticsearch.index.engine.Engine#FORCE_MERGE_UUID_KEY} after force merging it.
+     */
+    @Nullable
+    private final String forceMergeUUID;
+
     /**
      * Constructs a merge request over one or more indices.
      *
@@ -60,6 +72,7 @@ public static final class Defaults {
      */
     public ForceMergeRequest(String... indices) {
         super(indices);
+        forceMergeUUID = UUIDs.randomBase64UUID();
     }
 
     public ForceMergeRequest(StreamInput in) throws IOException {
@@ -67,6 +80,11 @@ public ForceMergeRequest(StreamInput in) throws IOException {
         maxNumSegments = in.readInt();
         onlyExpungeDeletes = in.readBoolean();
         flush = in.readBoolean();
+        if (in.getVersion().onOrAfter(FORCE_MERGE_UUID_VERSION)) {
+            forceMergeUUID = in.readOptionalString();
+        } else {
+            forceMergeUUID = null;
+        }
     }
 
     /**
@@ -103,6 +121,15 @@ public ForceMergeRequest onlyExpungeDeletes(boolean onlyExpungeDeletes) {
         return this;
     }
 
+    /**
+     * Force merge UUID to use when force merging or {@code null} if not using one in a mixed version cluster containing nodes older than
+     * {@link #FORCE_MERGE_UUID_VERSION}.
+     */
+    @Nullable
+    public String forceMergeUUID() {
+        return forceMergeUUID;
+    }
+
     /**
      * Should flush be performed after the merge. Defaults to {@code true}.
      */
@@ -132,6 +159,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeInt(maxNumSegments);
         out.writeBoolean(onlyExpungeDeletes);
         out.writeBoolean(flush);
+        if (out.getVersion().onOrAfter(FORCE_MERGE_UUID_VERSION)) {
+            out.writeOptionalString(forceMergeUUID);
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
index 751276812bddc..9d1e5756c81d6 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
@@ -310,9 +310,7 @@ List<String> getIndices() {
 
         FieldCapabilities build(boolean withIndices) {
             final String[] indices;
-            /* Eclipse can't deal with o -> o.name, maybe because of
-             * https://bugs.eclipse.org/bugs/show_bug.cgi?id=511750 */
-            Collections.sort(indiceList, Comparator.comparing((IndexCaps o) -> o.name));
+            Collections.sort(indiceList, Comparator.comparing(o -> o.name));
             if (withIndices) {
                 indices = indiceList.stream()
                     .map(caps -> caps.name)
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
index 09bba6eb0da07..bcb16c8690b04 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
@@ -69,6 +69,13 @@ public void execute(SimulatePipelineRequest.Parsed request, ActionListener<Simul
             final AtomicInteger counter = new AtomicInteger();
             final List<SimulateDocumentResult> responses =
                 new CopyOnWriteArrayList<>(new SimulateDocumentBaseResult[request.getDocuments().size()]);
+
+            if (request.getDocuments().isEmpty()) {
+                l.onResponse(new SimulatePipelineResponse(request.getPipeline().getId(),
+                    request.isVerbose(), responses));
+                return;
+            }
+
             int iter = 0;
             for (IngestDocument ingestDocument : request.getDocuments()) {
                 final int index = iter;
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
index 949d32c8fe734..ba51378cfaff8 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -173,8 +173,15 @@ static Parsed parse(Map<String, Object> config, boolean verbose, IngestService i
     private static List<IngestDocument> parseDocs(Map<String, Object> config) {
         List<Map<String, Object>> docs =
             ConfigurationUtils.readList(null, null, config, Fields.DOCS);
+        if (docs.isEmpty()) {
+            throw new IllegalArgumentException("must specify at least one document in [docs]");
+        }
         List<IngestDocument> ingestDocumentList = new ArrayList<>();
-        for (Map<String, Object> dataMap : docs) {
+        for (Object object : docs) {
+            if ((object instanceof Map) ==  false) {
+                throw new IllegalArgumentException("malformed [docs] section, should include an inner object");
+            }
+            Map<String, Object> dataMap = (Map<String, Object>) object;
             Map<String, Object> document = ConfigurationUtils.readMap(null, null,
                 dataMap, Fields.SOURCE);
             String index = ConfigurationUtils.readStringOrIntProperty(null, null,
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 8caf615dc1d07..1d2339dd9e6fe 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.search.TransportSearchAction.SearchTimeProvider;
 import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
@@ -77,7 +78,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
     private final BiFunction<String, String, Transport.Connection> nodeIdToConnection;
     private final SearchTask task;
     private final SearchPhaseResults<Result> results;
-    private final long clusterStateVersion;
+    private final ClusterState clusterState;
     private final Map<String, AliasFilter> aliasFilter;
     private final Map<String, Float> concreteIndexBoosts;
     private final Map<String, Set<String>> indexRoutings;
@@ -89,7 +90,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
     private final SearchTimeProvider timeProvider;
     private final SearchResponse.Clusters clusters;
 
-    private final GroupShardsIterator<SearchShardIterator> toSkipShardsIts;
+    protected final GroupShardsIterator<SearchShardIterator> toSkipShardsIts;
     protected final GroupShardsIterator<SearchShardIterator> shardsIts;
     private final int expectedTotalOps;
     private final AtomicInteger totalOps = new AtomicInteger();
@@ -98,14 +99,14 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
     private final boolean throttleConcurrentRequests;
 
     AbstractSearchAsyncAction(String name, Logger logger, SearchTransportService searchTransportService,
-                                        BiFunction<String, String, Transport.Connection> nodeIdToConnection,
-                                        Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
-                                        Map<String, Set<String>> indexRoutings,
-                                        Executor executor, SearchRequest request,
-                                        ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
-                                        SearchTimeProvider timeProvider, long clusterStateVersion,
-                                        SearchTask task, SearchPhaseResults<Result> resultConsumer, int maxConcurrentRequestsPerNode,
-                                        SearchResponse.Clusters clusters) {
+                              BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                              Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                              Map<String, Set<String>> indexRoutings,
+                              Executor executor, SearchRequest request,
+                              ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
+                              SearchTimeProvider timeProvider, ClusterState clusterState,
+                              SearchTask task, SearchPhaseResults<Result> resultConsumer, int maxConcurrentRequestsPerNode,
+                              SearchResponse.Clusters clusters) {
         super(name);
         final List<SearchShardIterator> toSkipIterators = new ArrayList<>();
         final List<SearchShardIterator> iterators = new ArrayList<>();
@@ -134,7 +135,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
         this.task = task;
         this.listener = listener;
         this.nodeIdToConnection = nodeIdToConnection;
-        this.clusterStateVersion = clusterStateVersion;
+        this.clusterState = clusterState;
         this.concreteIndexBoosts = concreteIndexBoosts;
         this.aliasFilter = aliasFilter;
         this.indexRoutings = indexRoutings;
@@ -338,7 +339,7 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha
                 final String resultsFrom = results.getSuccessfulResults()
                     .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(","));
                 logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
-                    currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion);
+                    currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterState.version());
             }
             executePhase(nextPhase);
         }
@@ -374,6 +375,11 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
         // we do make sure to clean it on a successful response from a shard
         SearchShardTarget shardTarget = shardIt.newSearchShardTarget(nodeId);
         onShardFailure(shardIndex, shardTarget, e);
+        final ShardRouting nextShard = shardIt.nextOrNull();
+        final boolean lastShard = nextShard == null;
+        if (lastShard) {
+            onShardGroupFailure(shardIndex, shardTarget, e);
+        }
 
         if (totalOps.incrementAndGet() == expectedTotalOps) {
             if (logger.isDebugEnabled()) {
@@ -384,11 +390,8 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
                     logger.trace(new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e);
                 }
             }
-            onShardGroupFailure(shardIndex, e);
             onPhaseDone();
         } else {
-            final ShardRouting nextShard = shardIt.nextOrNull();
-            final boolean lastShard = nextShard == null;
             // trace log this exception
             logger.trace(() -> new ParameterizedMessage(
                 "{}: Failed to execute [{}] lastShard [{}]",
@@ -404,7 +407,6 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
                             shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e);
                     }
                 }
-                onShardGroupFailure(shardIndex, e);
             }
         }
     }
@@ -412,10 +414,11 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard,
     /**
      * Executed once for every {@link ShardId} that failed on all available shard routing.
      *
-     * @param shardIndex the shard target that failed
-     * @param exc the final failure reason
+     * @param shardIndex the shard index that failed
+     * @param shardTarget the last shard target for this failure
+     * @param exc the last failure reason
      */
-    protected void onShardGroupFailure(int shardIndex, Exception exc) {}
+    protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {}
 
     /**
      * Executed once for every failed shard level request. This method is invoked before the next replica is tried for the given
@@ -559,7 +562,7 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) {
             try {
                 SearchShardTarget searchShardTarget = entry.getSearchShardTarget();
                 Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
-                sendReleaseSearchContext(entry.getRequestId(), connection, searchShardTarget.getOriginalIndices());
+                sendReleaseSearchContext(entry.getContextId(), connection, searchShardTarget.getOriginalIndices());
             } catch (Exception inner) {
                 inner.addSuppressed(exception);
                 logger.trace("failed to release context", inner);
@@ -681,4 +684,8 @@ private synchronized Runnable tryQueue(Runnable runnable) {
             return toExecute;
         }
     }
+
+    protected ClusterState clusterState() {
+        return clusterState;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
index 9d8b159b8e014..70eeb2a811e5b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
@@ -21,6 +21,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.search.SearchService.CanMatchResponse;
@@ -31,11 +32,9 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.transport.Transport;
 
-import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
@@ -61,17 +60,17 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction<CanMa
     private final GroupShardsIterator<SearchShardIterator> shardsIts;
 
     CanMatchPreFilterSearchPhase(Logger logger, SearchTransportService searchTransportService,
-                                        BiFunction<String, String, Transport.Connection> nodeIdToConnection,
-                                        Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
-                                        Map<String, Set<String>> indexRoutings,
-                                        Executor executor, SearchRequest request,
-                                        ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
-                                        TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion,
-                                        SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory,
-                                        SearchResponse.Clusters clusters) {
+                                 BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                                 Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                                 Map<String, Set<String>> indexRoutings,
+                                 Executor executor, SearchRequest request,
+                                 ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
+                                 TransportSearchAction.SearchTimeProvider timeProvider, ClusterState clusterState,
+                                 SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory,
+                                 SearchResponse.Clusters clusters) {
         //We set max concurrent shard requests to the number of shards so no throttling happens for can_match requests
         super("can_match", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
-                executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task,
+                executor, request, listener, shardsIts, timeProvider, clusterState, task,
                 new CanMatchSearchPhaseResults(shardsIts.size()), shardsIts.size(), clusters);
         this.phaseFactory = phaseFactory;
         this.shardsIts = shardsIts;
@@ -127,7 +126,18 @@ private static List<SearchShardIterator> sortShards(GroupShardsIterator<SearchSh
     }
 
     private static boolean shouldSortShards(MinAndMax<?>[] minAndMaxes) {
-        return Arrays.stream(minAndMaxes).anyMatch(Objects::nonNull);
+        Class<?> clazz = null;
+        for (MinAndMax<?> minAndMax : minAndMaxes) {
+            if (clazz == null) {
+                clazz = minAndMax == null ? null : minAndMax.getMin().getClass();
+            } else if (minAndMax != null && clazz != minAndMax.getMin().getClass()) {
+                // we don't support sort values that mix different types (e.g.: long/double, numeric/keyword).
+                // TODO: we could fail the request because there is a high probability
+                //  that the merging of topdocs will fail later for the same reason ?
+                return false;
+            }
+        }
+        return clazz != null;
     }
 
     private static Comparator<Integer> shardComparator(GroupShardsIterator<SearchShardIterator> shardsIts,
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
index c33eecee8bc75..d0abf798501b3 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
@@ -111,7 +111,7 @@ void cleanScrollIds(List<ScrollIdForNode> parsedScrollIds) {
                     } else {
                         try {
                             Transport.Connection connection = searchTransportService.getConnection(target.getClusterAlias(), node);
-                            searchTransportService.sendFreeContext(connection, target.getScrollId(),
+                            searchTransportService.sendFreeContext(connection, target.getContextId(),
                                 ActionListener.wrap(freed -> onFreedContext(freed.isFreed()), e -> onFailedFreedContext(e, node)));
                         } catch (Exception e) {
                             onFailedFreedContext(e, node);
diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index b4d52fa418e10..8352469042a58 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.query.QuerySearchRequest;
@@ -72,13 +71,11 @@ public void run() throws IOException {
         final CountedCollector<SearchPhaseResult> counter = new CountedCollector<>(queryResult::consumeResult,
             resultList.size(),
             () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context);
-        final SearchSourceBuilder sourceBuilder = context.getRequest().source();
-        progressListener.notifyListShards(progressListener.searchShards(resultList), sourceBuilder == null || sourceBuilder.size() != 0);
         for (final DfsSearchResult dfsResult : resultList) {
             final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget();
             Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
             QuerySearchRequest querySearchRequest = new QuerySearchRequest(searchShardTarget.getOriginalIndices(),
-                    dfsResult.getRequestId(), dfs);
+                    dfsResult.getContextId(), dfs);
             final int shardIndex = dfsResult.getShardIndex();
             searchTransportService.sendExecuteQuery(connection, querySearchRequest, context.getTask(),
                 new SearchActionListener<QuerySearchResult>(searchShardTarget, shardIndex) {
@@ -96,14 +93,15 @@ protected void innerOnResponse(QuerySearchResult response) {
                     public void onFailure(Exception exception) {
                         try {
                             context.getLogger().debug(() -> new ParameterizedMessage("[{}] Failed to execute query phase",
-                                querySearchRequest.id()), exception);
-                            progressListener.notifyQueryFailure(shardIndex, exception);
+                                querySearchRequest.contextId()), exception);
+                            progressListener.notifyQueryFailure(shardIndex, searchShardTarget, exception);
                             counter.onFailure(shardIndex, searchShardTarget, exception);
                         } finally {
                             // the query might not have been executed at all (for example because thread pool rejected
                             // execution) and the search context that was created in dfs phase might not be released.
                             // release it again to be in the safe side
-                            context.sendReleaseSearchContext(querySearchRequest.id(), connection, searchShardTarget.getOriginalIndices());
+                            context.sendReleaseSearchContext(
+                                querySearchRequest.contextId(), connection, searchShardTarget.getOriginalIndices());
                         }
                     }
                 });
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 41d216072e4b2..9142053be91d2 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -22,7 +22,9 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
@@ -30,6 +32,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.Transport;
 
@@ -50,17 +53,21 @@ final class FetchSearchPhase extends SearchPhase {
     private final Logger logger;
     private final SearchPhaseResults<SearchPhaseResult> resultConsumer;
     private final SearchProgressListener progressListener;
+    private final ClusterState clusterState;
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
-                     SearchPhaseContext context) {
-        this(resultConsumer, searchPhaseController, context,
+                     SearchPhaseContext context,
+                     ClusterState clusterState) {
+        this(resultConsumer, searchPhaseController, context, clusterState,
             (response, scrollId) -> new ExpandSearchPhase(context, response, scrollId));
     }
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
-                     SearchPhaseContext context, BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory) {
+                     SearchPhaseContext context,
+                     ClusterState clusterState,
+                     BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory) {
         super("fetch");
         if (context.getNumShards() != resultConsumer.getNumShards()) {
             throw new IllegalStateException("number of shards must match the length of the query results but doesn't:"
@@ -74,6 +81,7 @@ final class FetchSearchPhase extends SearchPhase {
         this.logger = context.getLogger();
         this.resultConsumer = resultConsumer;
         this.progressListener = context.getTask().getProgressListener();
+        this.clusterState = clusterState;
     }
 
     @Override
@@ -97,8 +105,14 @@ public void onFailure(Exception e) {
     private void innerRun() throws IOException {
         final int numShards = context.getNumShards();
         final boolean isScrollSearch = context.getRequest().scroll() != null;
-        List<SearchPhaseResult> phaseResults = queryResults.asList();
-        String scrollId = isScrollSearch ? TransportSearchHelper.buildScrollId(queryResults) : null;
+        final List<SearchPhaseResult> phaseResults = queryResults.asList();
+        final String scrollId;
+        if (isScrollSearch) {
+            final boolean includeContextUUID = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0);
+            scrollId = TransportSearchHelper.buildScrollId(queryResults, includeContextUUID);
+        } else {
+            scrollId = null;
+        }
         final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = resultConsumer.reduce();
         final boolean queryAndFetchOptimization = queryResults.length() == 1;
         final Runnable finishPhase = ()
@@ -143,7 +157,7 @@ private void innerRun() throws IOException {
                         SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
                         Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(),
                             searchShardTarget.getNodeId());
-                        ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getRequestId(), i, entry,
+                        ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getContextId(), i, entry,
                             lastEmittedDocPerShard, searchShardTarget.getOriginalIndices());
                         executeFetch(i, searchShardTarget, counter, fetchSearchRequest, queryResult.queryResult(),
                             connection);
@@ -153,10 +167,10 @@ private void innerRun() throws IOException {
         }
     }
 
-    protected ShardFetchSearchRequest createFetchRequest(long queryId, int index, IntArrayList entry,
-                                                               ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices) {
+    protected ShardFetchSearchRequest createFetchRequest(SearchContextId contextId, int index, IntArrayList entry,
+                                                         ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices) {
         final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[index] : null;
-        return new ShardFetchSearchRequest(originalIndices, queryId, entry, lastEmittedDoc);
+        return new ShardFetchSearchRequest(originalIndices, contextId, entry, lastEmittedDoc);
     }
 
     private void executeFetch(final int shardIndex, final SearchShardTarget shardTarget,
@@ -178,7 +192,8 @@ public void innerOnResponse(FetchSearchResult result) {
                 @Override
                 public void onFailure(Exception e) {
                     try {
-                        logger.debug(() -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.id()), e);
+                        logger.debug(
+                            () -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.contextId()), e);
                         progressListener.notifyFetchFailure(shardIndex, e);
                         counter.onFailure(shardIndex, shardTarget, e);
                     } finally {
@@ -201,7 +216,7 @@ private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
             try {
                 SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
                 Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
-                context.sendReleaseSearchContext(queryResult.getRequestId(), connection, searchShardTarget.getOriginalIndices());
+                context.sendReleaseSearchContext(queryResult.getContextId(), connection, searchShardTarget.getOriginalIndices());
             } catch (Exception e) {
                 context.getLogger().trace("failed to release context", e);
             }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
index 18b61516897d1..d69a10334bd78 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
@@ -20,16 +20,17 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.search.internal.SearchContextId;
 
 class ScrollIdForNode {
     private final String node;
-    private final long scrollId;
+    private final SearchContextId contextId;
     private final String clusterAlias;
 
-    ScrollIdForNode(@Nullable String clusterAlias, String node, long scrollId) {
+    ScrollIdForNode(@Nullable String clusterAlias, String node, SearchContextId contextId) {
         this.node = node;
         this.clusterAlias = clusterAlias;
-        this.scrollId = scrollId;
+        this.contextId = contextId;
     }
 
     public String getNode() {
@@ -41,15 +42,15 @@ public String getClusterAlias() {
         return clusterAlias;
     }
 
-    public long getScrollId() {
-        return scrollId;
+    public SearchContextId getContextId() {
+        return contextId;
     }
 
     @Override
     public String toString() {
         return "ScrollIdForNode{" +
             "node='" + node + '\'' +
-            ", scrollId=" + scrollId +
+            ", scrollId=" + contextId +
             ", clusterAlias='" + clusterAlias + '\'' +
             '}';
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 0782fbb310b65..0eecfce9e1e56 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -21,8 +21,10 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.transport.Transport;
@@ -37,17 +39,23 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction
     private final SearchPhaseController searchPhaseController;
 
     SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService,
-            final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter,
-            final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
-            final SearchPhaseController searchPhaseController, final Executor executor,
-            final SearchRequest request, final ActionListener<SearchResponse> listener,
-            final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
-            final long clusterStateVersion, final SearchTask task, SearchResponse.Clusters clusters) {
+                                       final BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                                       final Map<String, AliasFilter> aliasFilter,
+                                       final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
+                                       final SearchPhaseController searchPhaseController, final Executor executor,
+                                       final SearchRequest request, final ActionListener<SearchResponse> listener,
+                                       final GroupShardsIterator<SearchShardIterator> shardsIts,
+                                       final TransportSearchAction.SearchTimeProvider timeProvider,
+                                       final ClusterState clusterState, final SearchTask task, SearchResponse.Clusters clusters) {
         super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
                 executor, request, listener,
-                shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()),
+                shardsIts, timeProvider, clusterState, task, new ArraySearchPhaseResults<>(shardsIts.size()),
                 request.getMaxConcurrentShardRequests(), clusters);
         this.searchPhaseController = searchPhaseController;
+        SearchProgressListener progressListener = task.getProgressListener();
+        SearchSourceBuilder sourceBuilder = request.source();
+        progressListener.notifyListShards(SearchProgressListener.buildSearchShards(this.shardsIts),
+            SearchProgressListener.buildSearchShards(toSkipShardsIts), clusters, sourceBuilder == null || sourceBuilder.size() != 0);
     }
 
     @Override
@@ -60,6 +68,6 @@ protected void executePhaseOnShard(final SearchShardIterator shardIt, final Shar
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<DfsSearchResult> results, final SearchPhaseContext context) {
         return new DfsQueryPhase(results.getAtomicArray(), searchPhaseController, (queryResults) ->
-            new FetchSearchPhase(queryResults, searchPhaseController, context), context);
+            new FetchSearchPhase(queryResults, searchPhaseController, context, clusterState()), context);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
index 994ef5553bbc5..e22104b8f70af 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.transport.Transport;
 
@@ -96,11 +97,11 @@ interface SearchPhaseContext extends Executor {
 
     /**
      * Releases a search context with the given context ID on the node the given connection is connected to.
-     * @see org.elasticsearch.search.query.QuerySearchResult#getRequestId()
-     * @see org.elasticsearch.search.fetch.FetchSearchResult#getRequestId()
+     * @see org.elasticsearch.search.query.QuerySearchResult#getContextId()
+     * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId()
      *
      */
-    default void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+    default void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
         if (connection != null) {
             getSearchTransport().sendFreeContext(connection, contextId, originalIndices);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 59a5082ffe922..423f930b78fbe 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -664,9 +664,9 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
                     }
                     numReducePhases++;
                     index = 1;
-                    if (hasAggs) {
-                        progressListener.notifyPartialReduce(progressListener.searchShards(processedShards),
-                            topDocsStats.getTotalHits(), aggsBuffer[0], numReducePhases);
+                    if (hasAggs || hasTopDocs) {
+                        progressListener.notifyPartialReduce(SearchProgressListener.buildSearchShards(processedShards),
+                            topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0] : null, numReducePhases);
                     }
                 }
                 final int i = index++;
@@ -695,8 +695,8 @@ private synchronized List<TopDocs> getRemainingTopDocs() {
         public ReducedQueryPhase reduce() {
             ReducedQueryPhase reducePhase = controller.reducedQueryPhase(results.asList(),
                 getRemainingAggs(), getRemainingTopDocs(), topDocsStats, numReducePhases, false, performFinalReduce);
-            progressListener.notifyReduce(progressListener.searchShards(results.asList()),
-                reducePhase.totalHits, reducePhase.aggregations);
+            progressListener.notifyFinalReduce(SearchProgressListener.buildSearchShards(results.asList()),
+                reducePhase.totalHits, reducePhase.aggregations, reducePhase.numReducePhases);
             return reducePhase;
         }
 
@@ -751,7 +751,8 @@ ReducedQueryPhase reduce() {
                 List<SearchPhaseResult> resultList = results.asList();
                 final ReducedQueryPhase reducePhase =
                     reducedQueryPhase(resultList, isScrollRequest, trackTotalHitsUpTo, request.isFinalReduce());
-                listener.notifyReduce(listener.searchShards(resultList), reducePhase.totalHits, reducePhase.aggregations);
+                listener.notifyFinalReduce(SearchProgressListener.buildSearchShards(resultList),
+                    reducePhase.totalHits, reducePhase.aggregations, reducePhase.numReducePhases);
                 return reducePhase;
             }
         };
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
index 87146719a0f52..997151160f96b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
@@ -23,6 +23,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.action.search.SearchResponse.Clusters;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
@@ -48,24 +49,27 @@ abstract class SearchProgressListener {
      * Executed when shards are ready to be queried.
      *
      * @param shards The list of shards to query.
+     * @param skippedShards The list of skipped shards.
+     * @param clusters The statistics for remote clusters included in the search.
      * @param fetchPhase <code>true</code> if the search needs a fetch phase, <code>false</code> otherwise.
      **/
-    public void onListShards(List<SearchShard> shards, boolean fetchPhase) {}
+    protected void onListShards(List<SearchShard> shards, List<SearchShard> skippedShards, Clusters clusters, boolean fetchPhase) {}
 
     /**
      * Executed when a shard returns a query result.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)} )}.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards} )}.
      */
-    public void onQueryResult(int shardIndex) {}
+    protected void onQueryResult(int shardIndex) {}
 
     /**
      * Executed when a shard reports a query failure.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
+     * @param shardTarget The last shard target that thrown an exception.
      * @param exc The cause of the failure.
      */
-    public void onQueryFailure(int shardIndex, Exception exc) {}
+    protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {}
 
     /**
      * Executed when a partial reduce is created. The number of partial reduce can be controlled via
@@ -74,9 +78,9 @@ public void onQueryFailure(int shardIndex, Exception exc) {}
      * @param shards The list of shards that are part of this reduce.
      * @param totalHits The total number of hits in this reduce.
      * @param aggs The partial result for aggregations.
-     * @param version The version number for this reduce.
+     * @param reducePhase The version number for this reduce.
      */
-    public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {}
+    protected void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {}
 
     /**
      * Executed once when the final reduce is created.
@@ -84,28 +88,29 @@ public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, Inter
      * @param shards The list of shards that are part of this reduce.
      * @param totalHits The total number of hits in this reduce.
      * @param aggs The final result for aggregations.
+     * @param reducePhase The version number for this reduce.
      */
-    public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {}
+    protected void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {}
 
     /**
      * Executed when a shard returns a fetch result.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
      */
-    public void onFetchResult(int shardIndex) {}
+    protected void onFetchResult(int shardIndex) {}
 
     /**
      * Executed when a shard reports a fetch failure.
      *
-     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}.
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
      * @param exc The cause of the failure.
      */
-    public void onFetchFailure(int shardIndex, Exception exc) {}
+    protected void onFetchFailure(int shardIndex, Exception exc) {}
 
-    final void notifyListShards(List<SearchShard> shards, boolean fetchPhase) {
+    final void notifyListShards(List<SearchShard> shards, List<SearchShard> skippedShards, Clusters clusters, boolean fetchPhase) {
         this.shards = shards;
         try {
-            onListShards(shards, fetchPhase);
+            onListShards(shards, skippedShards, clusters, fetchPhase);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on list shards"), e);
         }
@@ -120,26 +125,26 @@ final void notifyQueryResult(int shardIndex) {
         }
     }
 
-    final void notifyQueryFailure(int shardIndex, Exception exc) {
+    final void notifyQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
         try {
-            onQueryFailure(shardIndex, exc);
+            onQueryFailure(shardIndex, shardTarget, exc);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("[{}] Failed to execute progress listener on query failure",
                 shards.get(shardIndex)), e);
         }
     }
 
-    final void notifyPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {
+    final void notifyPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
         try {
-            onPartialReduce(shards, totalHits, aggs, version);
+            onPartialReduce(shards, totalHits, aggs, reducePhase);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on partial reduce"), e);
         }
     }
 
-    final void notifyReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {
+    protected final void notifyFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
         try {
-            onReduce(shards, totalHits, aggs);
+            onFinalReduce(shards, totalHits, aggs, reducePhase);
         } catch (Exception e) {
             logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on reduce"), e);
         }
@@ -163,7 +168,7 @@ final void notifyFetchFailure(int shardIndex, Exception exc) {
         }
     }
 
-    final List<SearchShard> searchShards(List<? extends SearchPhaseResult> results) {
+    static List<SearchShard> buildSearchShards(List<? extends SearchPhaseResult> results) {
         return results.stream()
             .filter(Objects::nonNull)
             .map(SearchPhaseResult::getSearchShardTarget)
@@ -171,14 +176,14 @@ final List<SearchShard> searchShards(List<? extends SearchPhaseResult> results)
             .collect(Collectors.toUnmodifiableList());
     }
 
-    final List<SearchShard> searchShards(SearchShardTarget[] results) {
+    static List<SearchShard> buildSearchShards(SearchShardTarget[] results) {
         return Arrays.stream(results)
             .filter(Objects::nonNull)
             .map(e -> new SearchShard(e.getClusterAlias(), e.getShardId()))
             .collect(Collectors.toUnmodifiableList());
     }
 
-    final List<SearchShard> searchShards(GroupShardsIterator<SearchShardIterator> its) {
+    static List<SearchShard> buildSearchShards(GroupShardsIterator<SearchShardIterator> its) {
         return StreamSupport.stream(its.spliterator(), false)
             .map(e -> new SearchShard(e.getClusterAlias(), e.shardId()))
             .collect(Collectors.toUnmodifiableList());
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index d5060b728347d..24345c606003a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -21,9 +21,11 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.transport.Transport;
@@ -39,22 +41,24 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<Se
     private final SearchProgressListener progressListener;
 
     SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService,
-            final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter,
-            final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
-            final SearchPhaseController searchPhaseController, final Executor executor,
-            final SearchRequest request, final ActionListener<SearchResponse> listener,
-            final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider,
-            long clusterStateVersion, SearchTask task, SearchResponse.Clusters clusters) {
+                                    final BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                                    final Map<String, AliasFilter> aliasFilter,
+                                    final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings,
+                                    final SearchPhaseController searchPhaseController, final Executor executor,
+                                    final SearchRequest request, final ActionListener<SearchResponse> listener,
+                                    final GroupShardsIterator<SearchShardIterator> shardsIts,
+                                    final TransportSearchAction.SearchTimeProvider timeProvider,
+                                    ClusterState clusterState, SearchTask task, SearchResponse.Clusters clusters) {
         super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
-                executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task,
+                executor, request, listener, shardsIts, timeProvider, clusterState, task,
                 searchPhaseController.newSearchPhaseResults(task.getProgressListener(), request, shardsIts.size()),
                 request.getMaxConcurrentShardRequests(), clusters);
         this.searchPhaseController = searchPhaseController;
         this.progressListener = task.getProgressListener();
         final SearchProgressListener progressListener = task.getProgressListener();
         final SearchSourceBuilder sourceBuilder = request.source();
-        progressListener.notifyListShards(progressListener.searchShards(this.shardsIts),
-            sourceBuilder == null || sourceBuilder.size() != 0);
+        progressListener.notifyListShards(SearchProgressListener.buildSearchShards(this.shardsIts),
+            SearchProgressListener.buildSearchShards(toSkipShardsIts), clusters, sourceBuilder == null || sourceBuilder.size() != 0);
     }
 
     protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard,
@@ -64,12 +68,12 @@ protected void executePhaseOnShard(final SearchShardIterator shardIt, final Shar
     }
 
     @Override
-    protected void onShardGroupFailure(int shardIndex, Exception exc) {
-        progressListener.notifyQueryFailure(shardIndex, exc);
+    protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
+        progressListener.notifyQueryFailure(shardIndex, shardTarget, exc);
     }
 
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, final SearchPhaseContext context) {
-        return new FetchSearchPhase(results, searchPhaseController, context);
+        return new FetchSearchPhase(results, searchPhaseController, context, clusterState());
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 8ba9a8c9f0bd7..96206aa4bcd58 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
@@ -56,9 +55,9 @@
  * @see org.elasticsearch.client.Client#search(SearchRequest)
  * @see SearchResponse
  */
-public final class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable {
+public class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable {
 
-    private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
+    public static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
 
     public static final int DEFAULT_PRE_FILTER_SHARD_SIZE = 128;
     public static final int DEFAULT_BATCHED_REDUCE_SIZE = 512;
@@ -560,7 +559,7 @@ public boolean isSuggestOnly() {
     }
 
     @Override
-    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+    public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
         // generating description in a lazy way since source can be quite big
         return new SearchTask(id, type, action, null, parentTaskId, headers) {
             @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index ceaee96f5c131..b8b791360d30c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -213,7 +213,7 @@ public SearchRequestBuilder setVersion(boolean version) {
         sourceBuilder().version(version);
         return this;
     }
-    
+
     /**
      * Should each {@link org.elasticsearch.search.SearchHit} be returned with the
      * sequence number and primary term of the last modification of the document.
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index cb36dbd0cd8f2..81d61f2996ef4 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -113,6 +113,10 @@ public RestStatus status() {
         return RestStatus.status(successfulShards, totalShards, shardFailures);
     }
 
+    public SearchResponseSections getInternalResponse() {
+        return internalResponse;
+    }
+
     /**
      * The search hits.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
index 46de6de8f87c9..edc3f1b96a143 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.Transport;
 
@@ -147,11 +148,11 @@ private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, fi
                 }
                 connection = getConnection(target.getClusterAlias(), node);
             } catch (Exception ex) {
-                onShardFailure("query", counter, target.getScrollId(),
+                onShardFailure("query", counter, target.getContextId(),
                     ex, null, () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
                 continue;
             }
-            final InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(target.getScrollId(), request);
+            final InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(target.getContextId(), request);
             // we can't create a SearchShardTarget here since we don't know the index and shard ID we are talking to
             // we only know the node and the search context ID. Yet, the response will contain the SearchShardTarget
             // from the target node instead...that's why we pass null here
@@ -191,7 +192,7 @@ protected void innerOnResponse(T result) {
 
                 @Override
                 public void onFailure(Exception t) {
-                    onShardFailure("query", counter, target.getScrollId(), t, null,
+                    onShardFailure("query", counter, target.getContextId(), t, null,
                         () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
                 }
             };
@@ -247,7 +248,7 @@ protected final void sendResponse(SearchPhaseController.ReducedQueryPhase queryP
         }
     }
 
-    protected void onShardFailure(String phaseName, final CountDown counter, final long searchId, Exception failure,
+    protected void onShardFailure(String phaseName, final CountDown counter, final SearchContextId searchId, Exception failure,
                                   @Nullable SearchShardTarget searchShardTarget,
                                   Supplier<SearchPhase> nextPhaseSupplier) {
         if (logger.isDebugEnabled()) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index df18296de2a4a..d87ef021bfa59 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -86,7 +86,7 @@ public void run() {
                     if (docIds != null) {
                         final QuerySearchResult querySearchResult = queryResults.get(index);
                         ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[index];
-                        ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.getRequestId(), docIds,
+                        ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.getContextId(), docIds,
                             lastEmittedDoc);
                         SearchShardTarget searchShardTarget = querySearchResult.getSearchShardTarget();
                         DiscoveryNode node = clusterNodeLookup.apply(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
@@ -104,7 +104,7 @@ protected void innerOnResponse(FetchSearchResult response) {
 
                                 @Override
                                 public void onFailure(Exception t) {
-                                    onShardFailure(getName(), counter, querySearchResult.getRequestId(),
+                                    onShardFailure(getName(), counter, querySearchResult.getContextId(),
                                         t, querySearchResult.getSearchShardTarget(),
                                         () -> sendResponsePhase(reducedQueryPhase, fetchResults));
                                 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java
index 16459d81885ce..8a57ce51a7466 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java
@@ -29,19 +29,18 @@
  * A class that encapsulates the {@link ShardId} and the cluster alias
  * of a shard used during the search action.
  */
-public class SearchShard implements Comparable<SearchShard> {
+public final class SearchShard implements Comparable<SearchShard> {
     @Nullable
     private final String clusterAlias;
     private final ShardId shardId;
 
-    SearchShard(@Nullable String clusterAlias, ShardId shardId) {
+    public SearchShard(@Nullable String clusterAlias, ShardId shardId) {
         this.clusterAlias = clusterAlias;
         this.shardId = shardId;
     }
 
     /**
-     * Return the cluster alias if the shard is on a remote cluster and <code>null</code>
-     * otherwise (local).
+     * Return the cluster alias if we are executing a cross cluster search request, <code>null</code> otherwise.
      */
     @Nullable
     public String getClusterAlias() {
@@ -51,7 +50,6 @@ public String getClusterAlias() {
     /**
      * Return the {@link ShardId} of this shard.
      */
-    @Nullable
     public ShardId getShardId() {
         return shardId;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java
index 4719c1fda9d53..abfc876ad6000 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java
@@ -40,5 +40,4 @@ public SearchShardTask(long id, String type, String action, String description,
     public boolean shouldCancelChildrenOnCancellation() {
         return false;
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
index 97247e443bb64..c5a918c06f1bb 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
@@ -37,14 +37,14 @@ public SearchTask(long id, String type, String action, String description, TaskI
     /**
      * Attach a {@link SearchProgressListener} to this task.
      */
-    public void setProgressListener(SearchProgressListener progressListener) {
+    public final void setProgressListener(SearchProgressListener progressListener) {
         this.progressListener = progressListener;
     }
 
     /**
      * Return the {@link SearchProgressListener} attached to this task.
      */
-    public SearchProgressListener getProgressListener() {
+    public final SearchProgressListener getProgressListener() {
         return progressListener;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 16e8c17688906..681177e4399cd 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -87,7 +88,7 @@ public SearchTransportService(TransportService transportService,
         this.responseWrapper = responseWrapper;
     }
 
-    public void sendFreeContext(Transport.Connection connection, final long contextId, OriginalIndices originalIndices) {
+    public void sendFreeContext(Transport.Connection connection, final SearchContextId contextId, OriginalIndices originalIndices) {
         transportService.sendRequest(connection, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(originalIndices, contextId),
             TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(new ActionListener<SearchFreeContextResponse>() {
                 @Override
@@ -102,7 +103,8 @@ public void onFailure(Exception e) {
             }, SearchFreeContextResponse::new));
     }
 
-    public void sendFreeContext(Transport.Connection connection, long contextId, final ActionListener<SearchFreeContextResponse> listener) {
+    public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
+                                ActionListener<SearchFreeContextResponse> listener) {
         transportService.sendRequest(connection, FREE_CONTEXT_SCROLL_ACTION_NAME, new ScrollFreeContextRequest(contextId),
             TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(listener, SearchFreeContextResponse::new));
     }
@@ -194,39 +196,33 @@ public Map<String, Long> getPendingSearchRequests() {
     }
 
     static class ScrollFreeContextRequest extends TransportRequest {
-        private long id;
+        private SearchContextId contextId;
 
-        ScrollFreeContextRequest() {
-        }
-
-        ScrollFreeContextRequest(long id) {
-            this.id = id;
+        ScrollFreeContextRequest(SearchContextId contextId) {
+            this.contextId = contextId;
         }
 
         ScrollFreeContextRequest(StreamInput in) throws IOException {
             super(in);
-            id = in.readLong();
+            contextId = new SearchContextId(in);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeLong(id);
+            contextId.writeTo(out);
         }
 
-        public long id() {
-            return this.id;
+        public SearchContextId id() {
+            return this.contextId;
         }
 
-        }
+    }
 
     static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest {
         private OriginalIndices originalIndices;
 
-        SearchFreeContextRequest() {
-        }
-
-        SearchFreeContextRequest(OriginalIndices originalIndices, long id) {
+        SearchFreeContextRequest(OriginalIndices originalIndices, SearchContextId id) {
             super(id);
             this.originalIndices = originalIndices;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 2f7e8e338ea24..d7f60de9cffc0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -511,7 +511,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         BiFunction<String, String, Transport.Connection> connectionLookup = buildConnectionLookup(searchRequest.getLocalClusterAlias(),
             nodes::get, remoteConnections, searchTransportService::getConnection);
         boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators);
-        searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState.version(),
+        searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState,
             Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, routingMap, listener, preFilterSearchShards, clusters).start();
     }
 
@@ -560,7 +560,7 @@ private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction
                                                         GroupShardsIterator<SearchShardIterator> shardIterators,
                                                         SearchTimeProvider timeProvider,
                                                         BiFunction<String, String, Transport.Connection> connectionLookup,
-                                                        long clusterStateVersion,
+                                                        ClusterState clusterState,
                                                         Map<String, AliasFilter> aliasFilter,
                                                         Map<String, Float> concreteIndexBoosts,
                                                         Map<String, Set<String>> indexRoutings,
@@ -571,14 +571,14 @@ private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction
         if (preFilter) {
             return new CanMatchPreFilterSearchPhase(logger, searchTransportService, connectionLookup,
                 aliasFilter, concreteIndexBoosts, indexRoutings, executor, searchRequest, listener, shardIterators,
-                timeProvider, clusterStateVersion, task, (iter) -> {
+                timeProvider, clusterState, task, (iter) -> {
                 AbstractSearchAsyncAction<? extends SearchPhaseResult> action = searchAsyncAction(
                     task,
                     searchRequest,
                     iter,
                     timeProvider,
                     connectionLookup,
-                    clusterStateVersion,
+                    clusterState,
                     aliasFilter,
                     concreteIndexBoosts,
                     indexRoutings,
@@ -598,12 +598,12 @@ public void run() {
                 case DFS_QUERY_THEN_FETCH:
                     searchAsyncAction = new SearchDfsQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
                         aliasFilter, concreteIndexBoosts, indexRoutings, searchPhaseController, executor, searchRequest, listener,
-                        shardIterators, timeProvider, clusterStateVersion, task, clusters);
+                        shardIterators, timeProvider, clusterState, task, clusters);
                     break;
                 case QUERY_THEN_FETCH:
                     searchAsyncAction = new SearchQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
                         aliasFilter, concreteIndexBoosts, indexRoutings, searchPhaseController, executor, searchRequest, listener,
-                        shardIterators, timeProvider, clusterStateVersion, task, clusters);
+                        shardIterators, timeProvider, clusterState, task, clusters);
                     break;
                 default:
                     throw new IllegalStateException("Unknown search type: [" + searchRequest.searchType() + "]");
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index c848e227af4c0..f3755180b1e62 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
@@ -32,16 +33,25 @@
 
 final class TransportSearchHelper {
 
-    static InternalScrollSearchRequest internalScrollSearchRequest(long id, SearchScrollRequest request) {
+    private static final String INCLUDE_CONTEXT_UUID = "include_context_uuid";
+
+    static InternalScrollSearchRequest internalScrollSearchRequest(SearchContextId id, SearchScrollRequest request) {
         return new InternalScrollSearchRequest(request, id);
     }
 
-    static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
+    static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults,
+                                boolean includeContextUUID) throws IOException {
         try (RAMOutputStream out = new RAMOutputStream()) {
+            if (includeContextUUID) {
+                out.writeString(INCLUDE_CONTEXT_UUID);
+            }
             out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE);
             out.writeVInt(searchPhaseResults.asList().size());
             for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) {
-                out.writeLong(searchPhaseResult.getRequestId());
+                if (includeContextUUID) {
+                    out.writeString(searchPhaseResult.getContextId().getReaderId());
+                }
+                out.writeLong(searchPhaseResult.getContextId().getId());
                 SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget();
                 if (searchShardTarget.getClusterAlias() != null) {
                     out.writeString(
@@ -60,9 +70,19 @@ static ParsedScrollId parseScrollId(String scrollId) {
         try {
             byte[] bytes = Base64.getUrlDecoder().decode(scrollId);
             ByteArrayDataInput in = new ByteArrayDataInput(bytes);
-            String type = in.readString();
+            final boolean includeContextUUID;
+            final String type;
+            final String firstChunk = in.readString();
+            if (INCLUDE_CONTEXT_UUID.equals(firstChunk)) {
+                includeContextUUID = true;
+                type = in.readString();
+            } else {
+                includeContextUUID = false;
+                type = firstChunk;
+            }
             ScrollIdForNode[] context = new ScrollIdForNode[in.readVInt()];
             for (int i = 0; i < context.length; ++i) {
+                final String contextUUID = includeContextUUID ? in.readString() : "";
                 long id = in.readLong();
                 String target = in.readString();
                 String clusterAlias;
@@ -73,7 +93,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
                     clusterAlias = target.substring(0, index);
                     target = target.substring(index+1);
                 }
-                context[i] = new ScrollIdForNode(clusterAlias, target, id);
+                context[i] = new ScrollIdForNode(clusterAlias, target, new SearchContextId(contextUUID, id));
             }
             if (in.getPosition() != bytes.length) {
                 throw new IllegalArgumentException("Not all bytes were read");
diff --git a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
index 40bbf81534b58..cf4ab92baa0c6 100644
--- a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
+++ b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
@@ -23,12 +23,6 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchTask;
-import org.elasticsearch.action.search.SearchProgressActionListener;
-import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.support.AbstractClient;
@@ -108,38 +102,6 @@ > Task executeLocally(ActionType<Response> action, Request request, TaskListener
             listener::onResponse, listener::onFailure);
     }
 
-    /**
-     * Execute a {@link SearchRequest} locally and track the progress of the request through
-     * a {@link SearchProgressActionListener}.
-     */
-    public SearchTask executeSearchLocally(SearchRequest request, SearchProgressActionListener listener) {
-        // we cannot track the progress if remote cluster requests are splitted.
-        request.setCcsMinimizeRoundtrips(false);
-        TransportSearchAction action = (TransportSearchAction) actions.get(SearchAction.INSTANCE);
-        SearchTask task = (SearchTask) taskManager.register("transport", action.actionName, request);
-        task.setProgressListener(listener);
-        action.execute(task, request, new ActionListener<>() {
-            @Override
-            public void onResponse(SearchResponse response) {
-                try {
-                    taskManager.unregister(task);
-                } finally {
-                    listener.onResponse(response);
-                }
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                try {
-                    taskManager.unregister(task);
-                } finally {
-                    listener.onFailure(e);
-                }
-            }
-        });
-        return task;
-    }
-
     /**
      * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
      * {@link #executeLocally(ActionType, ActionRequest, TaskListener)}.
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index e445615e0fc73..70138ecc981bc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -127,6 +128,8 @@ public static List<Entry> getNamedWriteables() {
         registerMetaDataCustom(entries, IndexGraveyard.TYPE, IndexGraveyard::new, IndexGraveyard::readDiffFrom);
         registerMetaDataCustom(entries, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new,
             PersistentTasksCustomMetaData::readDiffFrom);
+        registerMetaDataCustom(entries, ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new,
+            ComponentTemplateMetadata::readDiffFrom);
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
         return entries;
@@ -145,6 +148,8 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
             IndexGraveyard::fromXContent));
         entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE),
             PersistentTasksCustomMetaData::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(ComponentTemplateMetadata.TYPE),
+            ComponentTemplateMetadata::fromXContent));
         return entries;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index b004af9d38ab6..acd3294f38641 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -304,20 +304,32 @@ private class ConnectionTarget {
             @Override
             protected void doRun() {
                 assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
-                transportService.connectToNode(discoveryNode, new ActionListener<Void>() {
-                    @Override
-                    public void onResponse(Void aVoid) {
-                        assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
-                        consecutiveFailureCount.set(0);
-                        logger.debug("connected to {}", discoveryNode);
-                        onCompletion(ActivityType.CONNECTING, null, disconnectActivity);
-                    }
+                if (transportService.nodeConnected(discoveryNode)) {
+                    // transportService.connectToNode is a no-op if already connected, but we don't want any DEBUG logging in this case
+                    // since we run this for every node on every cluster state update.
+                    logger.trace("still connected to {}", discoveryNode);
+                    onConnected();
+                } else {
+                    logger.debug("connecting to {}", discoveryNode);
+                    transportService.connectToNode(discoveryNode, new ActionListener<Void>() {
+                        @Override
+                        public void onResponse(Void aVoid) {
+                            assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
+                            logger.debug("connected to {}", discoveryNode);
+                            onConnected();
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            abstractRunnable.onFailure(e);
+                        }
+                    });
+                }
+            }
 
-                    @Override
-                    public void onFailure(Exception e) {
-                        abstractRunnable.onFailure(e);
-                    }
-                });
+            private void onConnected() {
+                consecutiveFailureCount.set(0);
+                onCompletion(ActivityType.CONNECTING, null, disconnectActivity);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
index 16e75464caf0f..479de5db45a18 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
@@ -495,6 +495,8 @@ public void onFailure(Exception e) {
     private void processJoinRequest(JoinRequest joinRequest, JoinHelper.JoinCallback joinCallback) {
         final Optional<Join> optionalJoin = joinRequest.getOptionalJoin();
         synchronized (mutex) {
+            updateMaxTermSeen(joinRequest.getTerm());
+
             final CoordinationState coordState = coordinationState.get();
             final boolean prevElectionWon = coordState.electionWon();
 
@@ -1115,7 +1117,7 @@ private class CoordinatorPeerFinder extends PeerFinder {
         protected void onActiveMasterFound(DiscoveryNode masterNode, long term) {
             synchronized (mutex) {
                 ensureTermAtLeast(masterNode, term);
-                joinHelper.sendJoinRequest(masterNode, joinWithDestination(lastJoin, masterNode, term));
+                joinHelper.sendJoinRequest(masterNode, getCurrentTerm(), joinWithDestination(lastJoin, masterNode, term));
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
index 2e47acf07b215..03f7b37d2b835 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
@@ -130,7 +130,7 @@ public ClusterTasksResult<JoinTaskExecutor.Task> execute(ClusterState currentSta
             StartJoinRequest::new,
             (request, channel, task) -> {
                 final DiscoveryNode destination = request.getSourceNode();
-                sendJoinRequest(destination, Optional.of(joinLeaderInTerm.apply(request)));
+                sendJoinRequest(destination, currentTermSupplier.getAsLong(), Optional.of(joinLeaderInTerm.apply(request)));
                 channel.sendResponse(Empty.INSTANCE);
             });
 
@@ -230,9 +230,9 @@ void logLastFailedJoinAttempt() {
         }
     }
 
-    public void sendJoinRequest(DiscoveryNode destination, Optional<Join> optionalJoin) {
+    public void sendJoinRequest(DiscoveryNode destination, long term, Optional<Join> optionalJoin) {
         assert destination.isMasterNode() : "trying to join master-ineligible " + destination;
-        final JoinRequest joinRequest = new JoinRequest(transportService.getLocalNode(), optionalJoin);
+        final JoinRequest joinRequest = new JoinRequest(transportService.getLocalNode(), term, optionalJoin);
         final Tuple<DiscoveryNode, JoinRequest> dedupKey = Tuple.tuple(destination, joinRequest);
         if (pendingOutgoingJoins.add(dedupKey)) {
             logger.debug("attempting to join {} with {}", destination, joinRequest);
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java
index 091a6809c84dc..fe1ce7d23ca44 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java
@@ -18,29 +18,53 @@
  */
 package org.elasticsearch.cluster.coordination;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
+import java.util.Objects;
 import java.util.Optional;
 
 public class JoinRequest extends TransportRequest {
 
+    /**
+     * The sending (i.e. joining) node.
+     */
     private final DiscoveryNode sourceNode;
 
+    /**
+     * The minimum term for which the joining node will accept any cluster state publications. If the joining node is in a strictly greater
+     * term than the master it wants to join then the master must enter a new term and hold another election. Doesn't necessarily match
+     * {@link JoinRequest#optionalJoin}.
+     */
+    private final long minimumTerm;
+
+    /**
+     * A vote for the receiving node. This vote is optional since the sending node may have voted for a different master in this term.
+     * That's ok, the sender likely discovered that the master we voted for lost the election and now we're trying to join the winner. Once
+     * the sender has successfully joined the master, the lack of a vote in its term causes another election (see
+     * {@link Publication#onMissingJoin(DiscoveryNode)}).
+     */
     private final Optional<Join> optionalJoin;
 
-    public JoinRequest(DiscoveryNode sourceNode, Optional<Join> optionalJoin) {
+    public JoinRequest(DiscoveryNode sourceNode, long minimumTerm, Optional<Join> optionalJoin) {
         assert optionalJoin.isPresent() == false || optionalJoin.get().getSourceNode().equals(sourceNode);
         this.sourceNode = sourceNode;
+        this.minimumTerm = minimumTerm;
         this.optionalJoin = optionalJoin;
     }
 
     public JoinRequest(StreamInput in) throws IOException {
         super(in);
         sourceNode = new DiscoveryNode(in);
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+            minimumTerm = in.readLong();
+        } else {
+            minimumTerm = 0L;
+        }
         optionalJoin = Optional.ofNullable(in.readOptionalWriteable(Join::new));
     }
 
@@ -48,6 +72,9 @@ public JoinRequest(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         sourceNode.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+            out.writeLong(minimumTerm);
+        }
         out.writeOptionalWriteable(optionalJoin.orElse(null));
     }
 
@@ -55,6 +82,17 @@ public DiscoveryNode getSourceNode() {
         return sourceNode;
     }
 
+    public long getMinimumTerm() {
+        return minimumTerm;
+    }
+
+    public long getTerm() {
+        // If the join is also present then its term will normally equal the corresponding term, but we do not require callers to
+        // obtain the term and the join in a synchronized fashion so it's possible that they disagree. Also older nodes do not share the
+        // minimum term, so for BWC we can take it from the join if present.
+        return Math.max(minimumTerm, optionalJoin.map(Join::getTerm).orElse(0L));
+    }
+
     public Optional<Join> getOptionalJoin() {
         return optionalJoin;
     }
@@ -66,21 +104,21 @@ public boolean equals(Object o) {
 
         JoinRequest that = (JoinRequest) o;
 
+        if (minimumTerm != that.minimumTerm) return false;
         if (!sourceNode.equals(that.sourceNode)) return false;
         return optionalJoin.equals(that.optionalJoin);
     }
 
     @Override
     public int hashCode() {
-        int result = sourceNode.hashCode();
-        result = 31 * result + optionalJoin.hashCode();
-        return result;
+        return Objects.hash(sourceNode, minimumTerm, optionalJoin);
     }
 
     @Override
     public String toString() {
         return "JoinRequest{" +
             "sourceNode=" + sourceNode +
+            ", minimumTerm=" + minimumTerm +
             ", optionalJoin=" + optionalJoin +
             '}';
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
new file mode 100644
index 0000000000000..2b160b1b2fb11
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A component template is a re-usable template as well as metadata about the template. Each
+ * component template is expected to be valid on its own. For example, if a component template
+ * contains a field "foo", it's expected to contain all the necessary settings/mappings/etc for the
+ * "foo" field. These component templates make up the individual pieces composing an index template.
+ */
+public class ComponentTemplate extends AbstractDiffable<ComponentTemplate> implements ToXContentObject {
+    private static final ParseField TEMPLATE = new ParseField("template");
+    private static final ParseField VERSION = new ParseField("version");
+    private static final ParseField METADATA = new ParseField("_meta");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ComponentTemplate, Void> PARSER =
+        new ConstructingObjectParser<>("component_template", false,
+            a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map<String, Object>) a[2]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Template.PARSER, TEMPLATE);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
+    }
+
+    private final Template template;
+    @Nullable
+    private final Long version;
+    @Nullable
+    private final Map<String, Object> metadata;
+
+    static Diff<ComponentTemplate> readComponentTemplateDiffFrom(StreamInput in) throws IOException {
+        return AbstractDiffable.readDiffFrom(ComponentTemplate::new, in);
+    }
+
+    public static ComponentTemplate parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ComponentTemplate(Template template, @Nullable Long version, @Nullable Map<String, Object> metadata) {
+        this.template = template;
+        this.version = version;
+        this.metadata = metadata;
+    }
+
+    public ComponentTemplate(StreamInput in) throws IOException {
+        this.template = new Template(in);
+        this.version = in.readOptionalVLong();
+        if (in.readBoolean()) {
+            this.metadata = in.readMap();
+        } else {
+            this.metadata = null;
+        }
+    }
+
+    public Template template() {
+        return template;
+    }
+
+    @Nullable
+    public Long version() {
+        return version;
+    }
+
+    @Nullable
+    public Map<String, Object> metadata() {
+        return metadata;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        this.template.writeTo(out);
+        out.writeOptionalVLong(this.version);
+        if (this.metadata == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            out.writeMap(this.metadata);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(template, version, metadata);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ComponentTemplate other = (ComponentTemplate) obj;
+        return Objects.equals(template, other.template) &&
+            Objects.equals(version, other.version) &&
+            Objects.equals(metadata, other.metadata);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TEMPLATE.getPreferredName(), this.template);
+        if (this.version != null) {
+            builder.field(VERSION.getPreferredName(), this.version);
+        }
+        if (this.metadata != null) {
+            builder.field(METADATA.getPreferredName(), this.metadata);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    static class Template extends AbstractDiffable<Template> implements ToXContentObject {
+        private static final ParseField SETTINGS = new ParseField("settings");
+        private static final ParseField MAPPINGS = new ParseField("mappings");
+        private static final ParseField ALIASES = new ParseField("aliases");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<Template, Void> PARSER = new ConstructingObjectParser<>("template", false,
+            a -> new Template((Settings) a[0], (CompressedXContent) a[1], (Map<String, AliasMetaData>) a[2]));
+
+        static {
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
+                new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered()))), MAPPINGS);
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+                Map<String, AliasMetaData> aliasMap = new HashMap<>();
+                while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    AliasMetaData alias = AliasMetaData.Builder.fromXContent(p);
+                    aliasMap.put(alias.alias(), alias);
+                }
+                return aliasMap;
+            }, ALIASES);
+        }
+
+        @Nullable
+        private final Settings settings;
+        @Nullable
+        private final CompressedXContent mappings;
+        @Nullable
+        private final Map<String, AliasMetaData> aliases;
+
+        Template(@Nullable Settings settings, @Nullable CompressedXContent mappings, @Nullable Map<String, AliasMetaData> aliases) {
+            this.settings = settings;
+            this.mappings = mappings;
+            this.aliases = aliases;
+        }
+
+        Template(StreamInput in) throws IOException {
+            if (in.readBoolean()) {
+                this.settings = Settings.readSettingsFromStream(in);
+            } else {
+                this.settings = null;
+            }
+            if (in.readBoolean()) {
+                this.mappings = CompressedXContent.readCompressedString(in);
+            } else {
+                this.mappings = null;
+            }
+            if (in.readBoolean()) {
+                this.aliases = in.readMap(StreamInput::readString, AliasMetaData::new);
+            } else {
+                this.aliases = null;
+            }
+        }
+
+        public Settings settings() {
+            return settings;
+        }
+
+        public CompressedXContent mappings() {
+            return mappings;
+        }
+
+        public Map<String, AliasMetaData> aliases() {
+            return aliases;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (this.settings == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                Settings.writeSettingsToStream(this.settings, out);
+            }
+            if (this.mappings == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                this.mappings.writeTo(out);
+            }
+            if (this.aliases == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                out.writeMap(this.aliases, StreamOutput::writeString, (stream, aliasMetaData) -> aliasMetaData.writeTo(stream));
+            }
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(settings, mappings, aliases);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Template other = (Template) obj;
+            return Objects.equals(settings, other.settings) &&
+                Objects.equals(mappings, other.mappings) &&
+                Objects.equals(aliases, other.aliases);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (this.settings != null) {
+                builder.startObject(SETTINGS.getPreferredName());
+                this.settings.toXContent(builder, params);
+                builder.endObject();
+            }
+            if (this.mappings != null) {
+                Map<String, Object> uncompressedMapping =
+                    XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2();
+                if (uncompressedMapping.size() > 0) {
+                    builder.field(MAPPINGS.getPreferredName());
+                    builder.map(uncompressedMapping);
+                }
+            }
+            if (this.aliases != null) {
+                builder.startObject(ALIASES.getPreferredName());
+                for (AliasMetaData alias : this.aliases.values()) {
+                    AliasMetaData.Builder.toXContent(alias, builder, params);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
new file mode 100644
index 0000000000000..1b1e6b346ef9a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * {@link ComponentTemplateMetadata} is a custom {@link MetaData} implementation for storing a map
+ * of component templates and their names.
+ */
+public class ComponentTemplateMetadata implements MetaData.Custom {
+    public static final String TYPE = "component_template";
+    private static final ParseField COMPONENT_TEMPLATE = new ParseField("component_template");
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ComponentTemplateMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE, false,
+        a -> new ComponentTemplateMetadata((Map<String, ComponentTemplate>) a[0]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
+            Map<String, ComponentTemplate> templates = new HashMap<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                String name = p.currentName();
+                templates.put(name, ComponentTemplate.parse(p));
+            }
+            return templates;
+        }, COMPONENT_TEMPLATE);
+    }
+    private final Map<String, ComponentTemplate> componentTemplates;
+
+    public ComponentTemplateMetadata(Map<String, ComponentTemplate> componentTemplates) {
+        this.componentTemplates = componentTemplates;
+    }
+
+    public ComponentTemplateMetadata(StreamInput in) throws IOException {
+        this.componentTemplates = in.readMap(StreamInput::readString, ComponentTemplate::new);
+    }
+
+    public Map<String, ComponentTemplate> componentTemplates() {
+        return this.componentTemplates;
+    }
+
+    @Override
+    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
+        return new ComponentTemplateMetadataDiff((ComponentTemplateMetadata) before, this);
+    }
+
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new ComponentTemplateMetadataDiff(in);
+    }
+
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_7_7_0;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(this.componentTemplates, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
+    }
+
+    public static ComponentTemplateMetadata fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(COMPONENT_TEMPLATE.getPreferredName());
+        for (Map.Entry<String, ComponentTemplate> template : componentTemplates.entrySet()) {
+            builder.field(template.getKey(), template.getValue());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.componentTemplates);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ComponentTemplateMetadata other = (ComponentTemplateMetadata) obj;
+        return Objects.equals(this.componentTemplates, other.componentTemplates);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    static class ComponentTemplateMetadataDiff implements NamedDiff<MetaData.Custom> {
+
+        final Diff<Map<String, ComponentTemplate>> componentTemplateDiff;
+
+        ComponentTemplateMetadataDiff(ComponentTemplateMetadata before, ComponentTemplateMetadata after) {
+            this.componentTemplateDiff = DiffableUtils.diff(before.componentTemplates, after.componentTemplates,
+                DiffableUtils.getStringKeySerializer());
+        }
+
+        ComponentTemplateMetadataDiff(StreamInput in) throws IOException {
+            this.componentTemplateDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                ComponentTemplate::new, ComponentTemplate::readComponentTemplateDiffFrom);
+        }
+
+        @Override
+        public MetaData.Custom apply(MetaData.Custom part) {
+            return new ComponentTemplateMetadata(componentTemplateDiff.apply(((ComponentTemplateMetadata) part).componentTemplates));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            componentTemplateDiff.writeTo(out);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 00d3a8d708d62..6154e6ecd675f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -73,6 +73,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
@@ -661,6 +662,12 @@ public ImmutableOpenMap<String, IndexTemplateMetaData> getTemplates() {
         return this.templates;
     }
 
+    public Map<String, ComponentTemplate> componentTemplates() {
+        return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
+            .map(ComponentTemplateMetadata::componentTemplates)
+            .orElse(Collections.emptyMap());
+    }
+
     public ImmutableOpenMap<String, Custom> customs() {
         return this.customs;
     }
@@ -1052,6 +1059,34 @@ public Builder templates(ImmutableOpenMap<String, IndexTemplateMetaData> templat
             return this;
         }
 
+        public Builder put(String name, ComponentTemplate componentTemplate) {
+            Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
+            // ಠ_ಠ at ImmutableOpenMap
+            Map<String, ComponentTemplate> existingTemplates =
+                Optional.ofNullable((ComponentTemplateMetadata) this.customs.get(ComponentTemplateMetadata.TYPE))
+                    .map(ctm -> new HashMap<>(ctm.componentTemplates()))
+                    .orElse(new HashMap<>());
+            existingTemplates.put(name, componentTemplate);
+            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(existingTemplates));
+            return this;
+        }
+
+        public Builder removeComponentTemplate(String name) {
+            // ಠ_ಠ at ImmutableOpenMap
+            Map<String, ComponentTemplate> existingTemplates =
+                Optional.ofNullable((ComponentTemplateMetadata) this.customs.get(ComponentTemplateMetadata.TYPE))
+                    .map(ctm -> new HashMap<>(ctm.componentTemplates()))
+                    .orElse(new HashMap<>());
+            existingTemplates.remove(name);
+            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(existingTemplates));
+            return this;
+        }
+
+        public Builder componentTemplates(Map<String, ComponentTemplate> componentTemplates) {
+            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(componentTemplates));
+            return this;
+        }
+
         public Custom getCustom(String type) {
             return customs.get(type);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index fd4e7023cfee9..6f8f680b10afd 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -318,12 +318,19 @@ public DiscoveryNode resolveNode(String node) {
     }
 
     /**
-     * resolves a set of node "descriptions" to concrete and existing node ids. "descriptions" can be (resolved in this order):
-     * - "_local" or "_master" for the relevant nodes
-     * - a node id
-     * - a wild card pattern that will be matched against node names
-     * - a "attr:value" pattern, where attr can be a node role (master, data, ingest etc.) in which case the value can be true or false,
-     *   or a generic node attribute name in which case value will be treated as a wildcard and matched against the node attribute values.
+     * Resolves a set of nodes according to the given sequence of node specifications. Implements the logic in various APIs that allow the
+     * user to run the action on a subset of the nodes in the cluster. See [Node specification] in the reference manual for full details.
+     *
+     * Works by tracking the current set of nodes and applying each node specification in sequence. The set starts out empty and each node
+     * specification may either add or remove nodes. For instance:
+     *
+     * - _local, _master and _all respectively add to the subset the local node, the currently-elected master, and all the nodes
+     * - node IDs, names, hostnames and IP addresses all add to the subset any nodes which match
+     * - a wildcard-based pattern of the form "attr*:value*" adds to the subset all nodes with a matching attribute with a matching value
+     * - role:true adds to the subset all nodes with a matching role
+     * - role:false removes from the subset all nodes with a matching role.
+     *
+     * An empty sequence of node specifications returns all nodes, since the corresponding actions run on all nodes by default.
      */
     public String[] resolveNodes(String... nodes) {
         if (nodes == null || nodes.length == 0) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
index 67122cb3ff1b1..4f7da131977fb 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
@@ -137,9 +137,6 @@ public static AllocationCommands fromXContent(XContentParser parser) throws IOEx
             throw new ElasticsearchParseException("No commands");
         }
         if (token == XContentParser.Token.FIELD_NAME) {
-            if (!parser.currentName().equals("commands")) {
-                throw new ElasticsearchParseException("expected field name to be named [commands], got [{}] instead", parser.currentName());
-            }
             if (!parser.currentName().equals("commands")) {
                 throw new ElasticsearchParseException("expected field name to be named [commands], got [{}] instead", parser.currentName());
             }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
index e78770ed2a85a..a8a44ecb6e925 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
@@ -92,11 +93,12 @@ private Collection<SpanQuery> collectTerms(IndexReader reader, MultiTermQuery qu
                         continue;
                     }
 
-                    final TermsEnum termsEnum = getTermsEnum(query, terms, null);
+                    final TermsEnum termsEnum = getTermsEnum(query, terms, new AttributeSource());
                     assert termsEnum != null;
 
-                    if (termsEnum == TermsEnum.EMPTY)
+                    if (termsEnum == TermsEnum.EMPTY) {
                         continue;
+                    }
 
                     BytesRef bytes;
                     while ((bytes = termsEnum.next()) != null) {
diff --git a/server/src/main/java/org/elasticsearch/common/util/BitArray.java b/server/src/main/java/org/elasticsearch/common/util/BitArray.java
index cfe2716e47e8e..06844fd99ec8c 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BitArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BitArray.java
@@ -32,6 +32,10 @@ public final class BitArray implements Releasable {
     private final BigArrays bigArrays;
     private LongArray bits;
 
+    /**
+     * Create the {@linkplain BitArray}.
+     * @param initialSize the initial size of underlying storage.
+     */
     public BitArray(int initialSize, BigArrays bigArrays) {
         this.bigArrays = bigArrays;
         this.bits = bigArrays.newLongArray(initialSize, true);
@@ -41,21 +45,31 @@ public BitArray(int initialSize, BigArrays bigArrays) {
      * Set the {@code index}th bit.
      */
     public void set(int index) {
-        fill(index, true);
+        int wordNum = wordNum(index);
+        bits = bigArrays.grow(bits, wordNum + 1);
+        bits.set(wordNum, bits.get(wordNum) | bitmask(index));
     }
 
     /**
      * Clear the {@code index}th bit.
      */
     public void clear(int index) {
-        fill(index, false);
+        int wordNum = wordNum(index);
+        if (wordNum >= bits.size()) {
+            /*
+             * No need to resize the array just to clear the bit because we'll
+             * initialize them to false when we grow the array anyway.
+             */
+            return;
+        }
+        bits.set(wordNum, bits.get(wordNum) & ~bitmask(index));
     }
 
     /**
      * Is the {@code index}th bit set?
      */
     public boolean get(int index) {
-        int wordNum = index >> 6;
+        int wordNum = wordNum(index);
         if (wordNum >= bits.size()) {
             /*
              * If the word is bigger than the array then it could *never* have
@@ -67,12 +81,12 @@ public boolean get(int index) {
         return (bits.get(wordNum) & bitmask) != 0;
     }
 
-    private void fill(int index, boolean bit) {
-        int wordNum = index >> 6;
-        bits = bigArrays.grow(bits,wordNum+1);
-        long bitmask = 1L << index;
-        long value = bit ? bits.get(wordNum) | bitmask : bits.get(wordNum) & ~bitmask;
-        bits.set(wordNum, value);
+    private static int wordNum(int index) {
+        return index >> 6;
+    }
+
+    private static long bitmask(int index) {
+        return 1L << index;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index a1319bcd59031..11f363f7ad706 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index de17eab3a4085..bd90f73054c94 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -104,6 +104,7 @@ public abstract class Engine implements Closeable {
 
     public static final String SYNC_COMMIT_ID = "sync_id"; // TODO: Remove sync_id in 9.0
     public static final String HISTORY_UUID_KEY = "history_uuid";
+    public static final String FORCE_MERGE_UUID_KEY = "force_merge_uuid";
     public static final String MIN_RETAINED_SEQNO = "min_retained_seq_no";
     public static final String MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID = "max_unsafe_auto_id_timestamp";
 
@@ -1035,18 +1036,12 @@ public final void flush() throws EngineException {
      */
     public abstract void rollTranslogGeneration() throws EngineException;
 
-    /**
-     * Force merges to 1 segment
-     */
-    public void forceMerge(boolean flush) throws IOException {
-        forceMerge(flush, 1, false, false, false);
-    }
-
     /**
      * Triggers a forced merge on this engine
      */
     public abstract void forceMerge(boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                                        boolean upgrade, boolean upgradeOnlyAncientSegments) throws EngineException, IOException;
+                                    boolean upgrade, boolean upgradeOnlyAncientSegments,
+                                    @Nullable String forceMergeUUID) throws EngineException, IOException;
 
     /**
      * Snapshots the most recent index and returns a handle to it. If needed will try and "commit" the
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 0a88123ba9413..a97f11af3213c 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -184,6 +184,12 @@ public class InternalEngine extends Engine {
     @Nullable
     private final String historyUUID;
 
+    /**
+     * UUID value that is updated every time the engine is force merged.
+     */
+    @Nullable
+    private volatile String forceMergeUUID;
+
     public InternalEngine(EngineConfig engineConfig) {
         this(engineConfig, LocalCheckpointTracker::new);
     }
@@ -222,7 +228,9 @@ public InternalEngine(EngineConfig engineConfig) {
                 this.localCheckpointTracker = createLocalCheckpointTracker(localCheckpointTrackerSupplier);
                 writer = createWriter();
                 bootstrapAppendOnlyInfoFromWriter(writer);
-                historyUUID = loadHistoryUUID(writer);
+                final Map<String, String> commitData = commitDataAsMap(writer);
+                historyUUID = loadHistoryUUID(commitData);
+                forceMergeUUID = commitData.get(FORCE_MERGE_UUID_KEY);
                 indexWriter = writer;
             } catch (IOException | TranslogCorruptedException e) {
                 throw new EngineCreationFailureException(shardId, "failed to create engine", e);
@@ -558,6 +566,12 @@ public String getHistoryUUID() {
         return historyUUID;
     }
 
+    /** returns the force merge uuid for the engine */
+    @Nullable
+    public String getForceMergeUUID() {
+        return forceMergeUUID;
+    }
+
     /** Returns how many bytes we are currently moving from indexing buffer to segments on disk */
     @Override
     public long getWritingBytes() {
@@ -567,8 +581,8 @@ public long getWritingBytes() {
     /**
      * Reads the current stored history ID from the IW commit data.
      */
-    private String loadHistoryUUID(final IndexWriter writer) {
-        final String uuid = commitDataAsMap(writer).get(HISTORY_UUID_KEY);
+    private String loadHistoryUUID(Map<String, String> commitData) {
+        final String uuid = commitData.get(HISTORY_UUID_KEY);
         if (uuid == null) {
             throw new IllegalStateException("commit doesn't contain history uuid");
         }
@@ -1815,7 +1829,8 @@ final Map<BytesRef, VersionValue> getVersionMap() {
 
     @Override
     public void forceMerge(final boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                           final boolean upgrade, final boolean upgradeOnlyAncientSegments) throws EngineException, IOException {
+                           final boolean upgrade, final boolean upgradeOnlyAncientSegments,
+                           final String forceMergeUUID) throws EngineException, IOException {
         if (onlyExpungeDeletes && maxNumSegments >= 0) {
             throw new IllegalArgumentException("only_expunge_deletes and max_num_segments are mutually exclusive");
         }
@@ -1850,6 +1865,7 @@ public void forceMerge(final boolean flush, int maxNumSegments, boolean onlyExpu
                     indexWriter.maybeMerge();
                 } else {
                     indexWriter.forceMerge(maxNumSegments, true /* blocks and waits for merges*/);
+                    this.forceMergeUUID = forceMergeUUID;
                 }
                 if (flush) {
                     flush(false, true);
@@ -2297,12 +2313,16 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl
                  * {@link IndexWriter#commit()} call flushes all documents, we defer computation of the maximum sequence number to the time
                  * of invocation of the commit data iterator (which occurs after all documents have been flushed to Lucene).
                  */
-                final Map<String, String> commitData = new HashMap<>(6);
+                final Map<String, String> commitData = new HashMap<>(7);
                 commitData.put(Translog.TRANSLOG_UUID_KEY, translog.getTranslogUUID());
                 commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(localCheckpoint));
                 commitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(localCheckpointTracker.getMaxSeqNo()));
                 commitData.put(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp.get()));
                 commitData.put(HISTORY_UUID_KEY, historyUUID);
+                final String currentForceMergeUUID = forceMergeUUID;
+                if (currentForceMergeUUID != null) {
+                    commitData.put(FORCE_MERGE_UUID_KEY, currentForceMergeUUID);
+                }
                 commitData.put(Engine.MIN_RETAINED_SEQNO, Long.toString(softDeletesPolicy.getMinRetainedSeqNo()));
                 logger.trace("committing writer with commit data [{}]", commitData);
                 return commitData.entrySet().iterator();
diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
index 7c39a3f11c4d5..0e96689d6324b 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
@@ -49,18 +50,19 @@
  */
 public final class NoOpEngine extends ReadOnlyEngine {
 
-    private final SegmentsStats stats;
+    private final SegmentsStats segmentsStats;
+    private final DocsStats docsStats;
 
     public NoOpEngine(EngineConfig config) {
         super(config, null, null, true, Function.identity());
-        this.stats = new SegmentsStats();
+        this.segmentsStats = new SegmentsStats();
         Directory directory = store.directory();
-        // Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
-        try (DirectoryReader reader = openDirectory(directory, false)) {
+        try (DirectoryReader reader = openDirectory(directory)) {
             for (LeafReaderContext ctx : reader.getContext().leaves()) {
                 SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
-                fillSegmentStats(segmentReader, true, stats);
+                fillSegmentStats(segmentReader, true, segmentsStats);
             }
+            this.docsStats = docsStats(reader);
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -117,7 +119,7 @@ public CacheHelper getReaderCacheHelper() {
     public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
         if (includeUnloadedSegments) {
             final SegmentsStats stats = new SegmentsStats();
-            stats.add(this.stats);
+            stats.add(this.segmentsStats);
             if (includeSegmentFileSizes == false) {
                 stats.clearFileSizes();
             }
@@ -127,6 +129,11 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
         }
     }
 
+    @Override
+    public DocsStats docStats() {
+        return docsStats;
+    }
+
     /**
      * This implementation will trim existing translog files using a {@link TranslogDeletionPolicy}
      * that retains nothing but the last translog generation from safe commit.
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index d84c7426447c8..62957e2c1c2c3 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
 import org.apache.lucene.search.ReferenceManager;
@@ -36,7 +35,6 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
@@ -76,7 +74,6 @@ public class ReadOnlyEngine extends Engine {
     private final ElasticsearchReaderManager readerManager;
     private final IndexCommit indexCommit;
     private final Lock indexWriterLock;
-    private final DocsStats docsStats;
     private final RamAccountingRefreshListener refreshListener;
     private final SafeCommitInfo safeCommitInfo;
     private final CompletionStatsCache completionStatsCache;
@@ -119,7 +116,6 @@ public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats
                 this.indexCommit = Lucene.getIndexCommit(lastCommittedSegmentInfos, directory);
                 reader = wrapReader(open(indexCommit), readerWrapperFunction);
                 readerManager = new ElasticsearchReaderManager(reader, refreshListener);
-                this.docsStats = docsStats(lastCommittedSegmentInfos);
                 assert translogStats != null || obtainLock : "mutiple translogs instances should not be opened at the same time";
                 this.translogStats = translogStats != null ? translogStats : translogStats(config, lastCommittedSegmentInfos);
                 this.indexWriterLock = indexWriterLock;
@@ -182,24 +178,6 @@ protected DirectoryReader open(IndexCommit commit) throws IOException {
         return new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(commit, OFF_HEAP_READER_ATTRIBUTES), Lucene.SOFT_DELETES_FIELD);
     }
 
-    private DocsStats docsStats(final SegmentInfos lastCommittedSegmentInfos) {
-        long numDocs = 0;
-        long numDeletedDocs = 0;
-        long sizeInBytes = 0;
-        if (lastCommittedSegmentInfos != null) {
-            for (SegmentCommitInfo segmentCommitInfo : lastCommittedSegmentInfos) {
-                numDocs += segmentCommitInfo.info.maxDoc() - segmentCommitInfo.getDelCount() - segmentCommitInfo.getSoftDelCount();
-                numDeletedDocs += segmentCommitInfo.getDelCount() + segmentCommitInfo.getSoftDelCount();
-                try {
-                    sizeInBytes += segmentCommitInfo.sizeInBytes();
-                } catch (IOException e) {
-                    throw new UncheckedIOException("Failed to get size for [" + segmentCommitInfo.info.name + "]", e);
-                }
-            }
-        }
-        return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
-    }
-
     @Override
     protected void closeNoLock(String reason, CountDownLatch closedLatch) {
         if (isClosed.compareAndSet(false, true)) {
@@ -388,7 +366,7 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException {
 
     @Override
     public void forceMerge(boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                           boolean upgrade, boolean upgradeOnlyAncientSegments) {
+                           boolean upgrade, boolean upgradeOnlyAncientSegments, String forceMergeUUID) {
     }
 
     @Override
@@ -463,11 +441,6 @@ public void trimOperationsFromTranslog(long belowTerm, long aboveSeqNo) {
     public void maybePruneDeletes() {
     }
 
-    @Override
-    public DocsStats docStats() {
-        return docsStats;
-    }
-
     @Override
     public void updateMaxUnsafeAutoIdTimestamp(long newTimestamp) {
 
@@ -511,13 +484,9 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) {
             maxSeqNoOfUpdatesOnPrimary + ">" + getMaxSeqNoOfUpdatesOrDeletes();
     }
 
-    protected static DirectoryReader openDirectory(Directory directory, boolean wrapSoftDeletes) throws IOException {
+    protected static DirectoryReader openDirectory(Directory directory) throws IOException {
         final DirectoryReader reader = DirectoryReader.open(directory, OFF_HEAP_READER_ATTRIBUTES);
-        if (wrapSoftDeletes) {
-            return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
-        } else {
-            return reader;
-        }
+        return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
index 27088382f2025..06352640dc162 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
@@ -21,8 +21,6 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedSetSortField;
-import org.apache.lucene.search.SortedSetSelector;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
@@ -54,20 +52,7 @@ public BinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exce
     public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested,
             boolean reverse) {
         XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
-        /**
-         * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
-         * returns a custom sort field otherwise.
-         */
-        if (nested != null ||
-                (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) ||
-                (source.sortMissingFirst(missingValue) == false && source.sortMissingLast(missingValue) == false)) {
-            return new SortField(getFieldName(), source, reverse);
-        }
-        SortField sortField = new SortedSetSortField(fieldName, reverse,
-            sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN);
-        sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ?
-            SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST);
-        return sortField;
+        return new SortField(getFieldName(), source, reverse);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
index 87d489d04b556..4e6b04a72418a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -97,6 +97,11 @@ public Instant toInstant(long value) {
             public Instant clampToValidRange(Instant instant) {
                 return instant;
             }
+
+            @Override
+            public long parsePointAsMillis(byte[] value) {
+                return LongPoint.decodeDimension(value, 0);
+            }
         },
         NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS) {
             @Override
@@ -113,6 +118,11 @@ public Instant toInstant(long value) {
             public Instant clampToValidRange(Instant instant) {
                 return DateUtils.clampToNanosRange(instant);
             }
+
+            @Override
+            public long parsePointAsMillis(byte[] value) {
+                return DateUtils.toMilliSeconds(LongPoint.decodeDimension(value, 0));
+            }
         };
 
         private final String type;
@@ -141,8 +151,17 @@ NumericType numericType() {
          */
         public abstract Instant toInstant(long value);
 
+        /**
+         * Return the instant that this range can represent that is closest to
+         * the provided instant.
+         */
         public abstract Instant clampToValidRange(Instant instant);
 
+        /**
+         * Decode the points representation of this field as milliseconds.
+         */
+        public abstract long parsePointAsMillis(byte[] value);
+
         public static Resolution ofOrdinal(int ord) {
             for (Resolution resolution : values()) {
                 if (ord == resolution.ordinal()) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 7fde4359742e3..b83427a53a3ed 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -204,6 +204,7 @@ protected KeywordFieldType(KeywordFieldType ref) {
             this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace;
         }
 
+        @Override
         public KeywordFieldType clone() {
             return new KeywordFieldType(this);
         }
@@ -321,8 +322,7 @@ protected KeywordFieldMapper(String simpleName, MappedFieldType fieldType, Mappe
 
     /** Values that have more chars than the return value of this method will
      *  be skipped at parsing time. */
-    // pkg-private for testing
-    int ignoreAbove() {
+    public int ignoreAbove() {
         return ignoreAbove;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index b2c536a900e33..d675735834f7b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -120,24 +120,6 @@ public IndexableField[] getFields(String name) {
             return f.toArray(new IndexableField[f.size()]);
         }
 
-        /**
-         * Returns an array of values of the field specified as the method parameter.
-         * This method returns an empty array when there are no
-         * matching fields.  It never returns null.
-         * If you want the actual numeric field instances back, use {@link #getFields}.
-         * @param name the name of the field
-         * @return a <code>String[]</code> of field values
-         */
-        public final String[] getValues(String name) {
-            List<String> result = new ArrayList<>();
-            for (IndexableField field : fields) {
-                if (field.name().equals(name) && field.stringValue() != null) {
-                    result.add(field.stringValue());
-                }
-            }
-            return result.toArray(new String[result.size()]);
-        }
-
         public IndexableField getField(String name) {
             for (IndexableField field : fields) {
                 if (field.name().equals(name)) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index 4ddda3df0af2d..05bf6b61d1de1 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -21,8 +21,6 @@
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
@@ -31,6 +29,7 @@
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -38,6 +37,8 @@
 import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 
@@ -47,6 +48,8 @@
  * can be implemented. */
 public abstract class StringFieldType extends TermBasedFieldType {
 
+    private static final Pattern WILDCARD_PATTERN = Pattern.compile("(\\\\.)|([?*]+)");
+
     public StringFieldType() {}
 
     protected StringFieldType(MappedFieldType ref) {
@@ -92,16 +95,41 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
 
     @Override
     public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        Query termQuery = termQuery(value, context);
-        if (termQuery instanceof MatchNoDocsQuery || termQuery instanceof MatchAllDocsQuery) {
-            return termQuery;
-        }
-
+        failIfNotIndexed();
         if (context.allowExpensiveQueries() == false) {
             throw new ElasticsearchException("[wildcard] queries cannot be executed when '" +
                     ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false.");
         }
-        Term term = MappedFieldType.extractTerm(termQuery);
+
+        Term term;
+        if (searchAnalyzer() != null) {
+            // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
+            // is a char_filter that would otherwise remove them
+            Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
+            BytesRefBuilder sb = new BytesRefBuilder();
+            int last = 0;
+
+            while (wildcardMatcher.find()) {
+                if (wildcardMatcher.start() > 0) {
+                    String chunk = value.substring(last, wildcardMatcher.start());
+
+                    BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
+                    sb.append(normalized);
+                }
+                // append the matched group - without normalizing
+                sb.append(new BytesRef(wildcardMatcher.group()));
+
+                last = wildcardMatcher.end();
+            }
+            if (last < value.length()) {
+                String chunk = value.substring(last);
+                BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
+                sb.append(normalized);
+            }
+            term = new Term(name(), sb.toBytesRef());
+        } else {
+            term = new Term(name(), indexedValueForSearch(value));
+        }
 
         WildcardQuery query = new WildcardQuery(term);
         QueryParsers.setRewriteMethod(query, method);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index 895243d43faf3..1f9ccfef04330 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -33,7 +33,6 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
 import org.apache.lucene.search.AutomatonQuery;
@@ -425,13 +424,13 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
 
         public IntervalsSource intervals(BytesRef term) {
             if (term.length > maxChars) {
-                return XIntervals.prefix(term);
+                return Intervals.prefix(term);
             }
             if (term.length >= minChars) {
                 return Intervals.fixField(name(), Intervals.term(term));
             }
             String wildcardTerm = term.utf8ToString() + "?".repeat(Math.max(0, minChars - term.length));
-            return Intervals.or(Intervals.fixField(name(), XIntervals.wildcard(new BytesRef(wildcardTerm))), Intervals.term(term));
+            return Intervals.or(Intervals.fixField(name(), Intervals.wildcard(new BytesRef(wildcardTerm))), Intervals.term(term));
         }
 
         @Override
@@ -675,7 +674,7 @@ public IntervalsSource intervals(String text, int maxGaps, boolean ordered,
                 if (prefixFieldType != null) {
                     return prefixFieldType.intervals(normalizedTerm);
                 }
-                return XIntervals.prefix(normalizedTerm);
+                return Intervals.prefix(normalizedTerm);
             }
             IntervalBuilder builder = new IntervalBuilder(name(), analyzer == null ? searchAnalyzer() : analyzer);
             return builder.analyzeText(text, maxGaps, ordered);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
index e5c33b83f24e5..d32d4fa9d3f11 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
@@ -30,13 +30,11 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -90,7 +88,7 @@ public MetadataFieldMapper getDefault(ParserContext context) {
         }
     }
 
-    public static final class TypeFieldType extends StringFieldType {
+    public static final class TypeFieldType extends ConstantFieldType {
 
         TypeFieldType() {
         }
@@ -121,61 +119,8 @@ public ValuesSourceType getValuesSourceType() {
         }
 
         @Override
-        public boolean isSearchable() {
-            return true;
-        }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new MatchAllDocsQuery();
-        }
-
-        @Override
-        public Query termQuery(Object value, QueryShardContext context) {
-            return termsQuery(Arrays.asList(value), context);
-        }
-
-        @Override
-        public Query termsQuery(List<?> values, QueryShardContext context) {
-            DocumentMapper mapper = context.getMapperService().documentMapper();
-            if (mapper == null) {
-                return new MatchNoDocsQuery("No types");
-            }
-            BytesRef indexType = indexedValueForSearch(mapper.type());
-            if (values.stream()
-                    .map(this::indexedValueForSearch)
-                    .anyMatch(indexType::equals)) {
-                if (context.getMapperService().hasNested()) {
-                    // type filters are expected not to match nested docs
-                    return Queries.newNonNestedFilter();
-                } else {
-                    return new MatchAllDocsQuery();
-                }
-            } else {
-                return new MatchNoDocsQuery("Type list does not contain the index type");
-            }
-        }
-
-        @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
-            Query result = new MatchAllDocsQuery();
-            String type = context.getMapperService().documentMapper().type();
-            if (type != null) {
-                BytesRef typeBytes = new BytesRef(type);
-                if (lowerTerm != null) {
-                    int comp = indexedValueForSearch(lowerTerm).compareTo(typeBytes);
-                    if (comp > 0 || (comp == 0 && includeLower == false)) {
-                        result = new MatchNoDocsQuery("[_type] was lexicographically smaller than lower bound of range");
-                    }
-                }
-                if (upperTerm != null) {
-                    int comp = indexedValueForSearch(upperTerm).compareTo(typeBytes);
-                    if (comp < 0 || (comp == 0 && includeUpper == false)) {
-                        result = new MatchNoDocsQuery("[_type] was lexicographically greater than upper bound of range");
-                    }
-                }
-            }
-            return result;
+        protected boolean matches(String pattern, QueryShardContext context) {
+            return pattern.equals(MapperService.SINGLE_MAPPING_NAME);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
index eefbc0cb3ae55..13c60c0a49f5e 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
@@ -21,7 +21,6 @@
 
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.FilteredIntervalsSource;
 import org.apache.lucene.queries.intervals.IntervalIterator;
 import org.apache.lucene.queries.intervals.Intervals;
@@ -655,13 +654,12 @@ public IntervalsSource getSource(QueryShardContext context, MappedFieldType fiel
                     analyzer = fieldType.searchAnalyzer();
                 }
                 BytesRef normalizedTerm = analyzer.normalize(useField, pattern);
-                // TODO Intervals.wildcard() should take BytesRef
-                source = Intervals.fixField(useField, XIntervals.wildcard(normalizedTerm));
+                source = Intervals.fixField(useField, Intervals.wildcard(normalizedTerm));
             }
             else {
                 checkPositions(fieldType);
                 BytesRef normalizedTerm = analyzer.normalize(fieldType.name(), pattern);
-                source = XIntervals.wildcard(normalizedTerm);
+                source = Intervals.wildcard(normalizedTerm);
             }
             return source;
         }
@@ -798,7 +796,7 @@ public IntervalsSource getSource(QueryShardContext context, MappedFieldType fiel
             FuzzyQuery fq = new FuzzyQuery(new Term(fieldType.name(), normalizedTerm),
                 fuzziness.asDistance(term), prefixLength, 128, transpositions);
             CompiledAutomaton[] automata = fq.getAutomata();
-            source = XIntervals.multiterm(automata[automata.length - 1], term);
+            source = Intervals.multiterm(automata[automata.length - 1], term);
             if (useField != null) {
                 source = Intervals.fixField(useField, source);
             }
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 4c7b8f568aff3..56707bc198beb 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1073,7 +1073,7 @@ public void forceMerge(ForceMergeRequest forceMerge) throws IOException {
         }
         Engine engine = getEngine();
         engine.forceMerge(forceMerge.flush(), forceMerge.maxNumSegments(),
-            forceMerge.onlyExpungeDeletes(), false, false);
+            forceMerge.onlyExpungeDeletes(), false, false, forceMerge.forceMergeUUID());
     }
 
     /**
@@ -1089,7 +1089,7 @@ public org.apache.lucene.util.Version upgrade(UpgradeRequest upgrade) throws IOE
         final Engine engine = getEngine();
         engine.forceMerge(true,  // we need to flush at the end to make sure the upgrade is durable
             Integer.MAX_VALUE, // we just want to upgrade the segments, not actually optimize to a single segment
-            false, true, upgrade.upgradeOnlyAncientSegments());
+            false, true, upgrade.upgradeOnlyAncientSegments(), null);
         org.apache.lucene.util.Version version = minimumCompatibleVersion();
         if (logger.isTraceEnabled()) {
             logger.trace("upgraded segments for {} from version {} to version {}", shardId, previousVersion, version);
diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index b84b6aa4c1e49..e6301371520a5 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -135,7 +136,7 @@ void recoverFromLocalShards(Consumer<MappingMetaData> mappingUpdateConsumer, fin
                     // just trigger a merge to do housekeeping on the
                     // copied segments - we will also see them in stats etc.
                     indexShard.getEngine().forceMerge(false, -1, false,
-                        false, false);
+                        false, false, UUIDs.randomBase64UUID());
                     return true;
                 } catch (IOException ex) {
                     throw new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", ex);
diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 400e5355c0942..65df5e2b35268 100644
--- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
@@ -56,6 +57,7 @@
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -313,7 +315,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
             else {
                 seenFields.add(field.name());
             }
-            String[] values = doc.getValues(field.name());
+            String[] values = getValues(doc.getFields(field.name()));
             documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values)));
         }
         return generateTermVectors(indexShard,
@@ -321,6 +323,25 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
                 request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
+    /**
+     * Returns an array of values of the field specified as the method parameter.
+     * This method returns an empty array when there are no
+     * matching fields.  It never returns null.
+     * @param fields The <code>IndexableField</code> to get the values from
+     * @return a <code>String[]</code> of field values
+     */
+    public static String[] getValues(IndexableField[] fields) {
+        List<String> result = new ArrayList<>();
+        for (IndexableField field : fields) {
+            if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) {
+                result.add(field.binaryValue().utf8ToString());
+            } else if (field.fieldType() instanceof StringFieldType) {
+                result.add(field.stringValue());
+            }
+        }
+        return result.toArray(new String[0]);
+    }
+
     private static ParsedDocument parseDocument(IndexShard indexShard, String index, BytesReference doc,
                                                 XContentType xContentType, String routing) {
         MapperService mapperService = indexShard.mapperService();
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index e03b0c98e0e98..90f20811c3dc9 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -21,6 +21,7 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
@@ -52,6 +53,7 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, final Node
         final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?
                 new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias"))) :
                 new GetAliasesRequest();
+        getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
         getAliasesRequest.local(request.paramAsBoolean("local", getAliasesRequest.local()));
 
         return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestResponseListener<GetAliasesResponse>(channel) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index a70fcec89b0c8..d760d56e8d542 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -90,7 +90,7 @@ protected void documentation(StringBuilder sb) {
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final IndicesOptions indicesOptions = IndicesOptions.strictExpand();
+        final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.strictExpand());
         final boolean local = request.paramAsBoolean("local", false);
         final TimeValue masterNodeTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT);
         final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false);
@@ -109,6 +109,12 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) {
                     final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(request, 4, listener);
                     groupedListener.onResponse(getSettingsResponse);
 
+                    // The list of indices that will be returned is determined by the indices returned from the Get Settings call.
+                    // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the
+                    // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so
+                    // force the IndicesOptions for all the sub-requests to be as inclusive as possible.
+                    final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden();
+
                     // Indices that were successfully resolved during the get settings request might be deleted when the subsequent cluster
                     // state, cluster health and indices stats requests execute. We have to distinguish two cases:
                     // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the
@@ -117,11 +123,11 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) {
                     //    fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
                     // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve the
                     // index names with the same indices options that we used for the initial cluster state request (strictExpand).
-                    sendIndicesStatsRequest(indices, indicesOptions, includeUnloadedSegments, client,
+                    sendIndicesStatsRequest(indices, subRequestIndicesOptions, includeUnloadedSegments, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
-                    sendClusterStateRequest(indices, indicesOptions, local, masterNodeTimeout, client,
+                    sendClusterStateRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
-                    sendClusterHealthRequest(indices, indicesOptions, local, masterNodeTimeout, client,
+                    sendClusterHealthRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
                 }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 266ff0321033a..b95aaf4b86018 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -164,7 +164,7 @@ public static void parseSearchRequest(SearchRequest searchRequest, RestRequest r
         searchRequest.routing(request.param("routing"));
         searchRequest.preference(request.param("preference"));
         searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
-        searchRequest.setCcsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", true));
+        searchRequest.setCcsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", searchRequest.isCcsMinimizeRoundtrips()));
 
         checkRestTotalHits(request, searchRequest);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 24ab7f5e8bf43..bed38deb28a29 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -62,6 +62,7 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
@@ -82,7 +83,7 @@
 
 final class DefaultSearchContext extends SearchContext {
 
-    private final long id;
+    private final SearchContextId id;
     private final ShardSearchRequest request;
     private final SearchShardTarget shardTarget;
     private final LongSupplier relativeTimeSupplier;
@@ -152,7 +153,7 @@ final class DefaultSearchContext extends SearchContext {
     private final QueryShardContext queryShardContext;
     private final FetchPhase fetchPhase;
 
-    DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
+    DefaultSearchContext(SearchContextId id, ShardSearchRequest request, SearchShardTarget shardTarget,
                          Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService,
                          IndexShard indexShard, BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout,
                          FetchPhase fetchPhase) throws IOException {
@@ -295,7 +296,7 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
     }
 
     @Override
-    public long id() {
+    public SearchContextId id() {
         return this.id;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
index b09c8983c1394..7f681bfa47c92 100644
--- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -201,6 +202,13 @@ public DateTime(StreamInput in) throws IOException {
             String zoneId = in.readString();
             this.timeZone = ZoneId.of(zoneId);
             this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt());
+            if (in.getVersion().onOrAfter(Version.V_7_7_0) && in.getVersion().before(Version.V_8_0_0)) {
+                /* when deserialising from 7.7+ nodes expect a flag indicating if a pattern is of joda style
+                   This is only used to support joda style indices in 7.x, in 8 we no longer support this.
+                   All indices in 8 should use java style pattern. Hence we can ignore this flag.
+                */
+                in.readBoolean();
+            }
         }
 
         @Override
@@ -213,6 +221,13 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(formatter.pattern());
             out.writeString(timeZone.getId());
             out.writeVInt(resolution.ordinal());
+            if (out.getVersion().onOrAfter(Version.V_7_7_0) && out.getVersion().before(Version.V_8_0_0)) {
+                /* when serializing to 7.7+  send out a flag indicating if a pattern is of joda style
+                   This is only used to support joda style indices in 7.x, in 8 we no longer support this.
+                   All indices in 8 should use java style pattern. Hence this flag is always false.
+                */
+                out.writeBoolean(false);
+            }
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java b/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
index 9e8d41e2ae29c..03ac85a8d81ac 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
@@ -23,20 +23,21 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.internal.SearchContextId;
 
 import java.io.IOException;
 
 public class SearchContextMissingException extends ElasticsearchException {
 
-    private final long id;
+    private final SearchContextId contextId;
 
-    public SearchContextMissingException(long id) {
-        super("No search context found for id [" + id + "]");
-        this.id = id;
+    public SearchContextMissingException(SearchContextId contextId) {
+        super("No search context found for id [" + contextId.getId() + "]");
+        this.contextId = contextId;
     }
 
-    public long id() {
-        return this.id;
+    public SearchContextId contextId() {
+        return this.contextId;
     }
 
     @Override
@@ -46,12 +47,12 @@ public RestStatus status() {
 
     public SearchContextMissingException(StreamInput in) throws IOException{
         super(in);
-        id = in.readLong();
+        contextId = new SearchContextId(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeLong(id);
+        contextId.writeTo(out);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
index 77af879831e02..15fab25f01b6b 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.TransportResponse;
 
@@ -39,7 +40,7 @@ public abstract class SearchPhaseResult extends TransportResponse {
 
     private SearchShardTarget searchShardTarget;
     private int shardIndex = -1;
-    protected long requestId;
+    protected SearchContextId contextId;
 
     protected SearchPhaseResult() {
 
@@ -50,10 +51,10 @@ protected SearchPhaseResult(StreamInput in) throws IOException {
     }
 
     /**
-     * Returns the results request ID that is used to reference the search context on the executing node
+     * Returns the search context ID that is used to reference the search context on the executing node
      */
-    public long getRequestId() {
-        return requestId;
+    public SearchContextId getContextId() {
+        return contextId;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 49b560dcd5b58..c7fa46bf9a353 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -83,6 +84,7 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
@@ -284,7 +286,7 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
     }
 
     protected void putContext(SearchContext context) {
-        final SearchContext previous = activeContexts.put(context.id(), context);
+        final SearchContext previous = activeContexts.put(context.id().getId(), context);
         assert previous == null;
     }
 
@@ -412,8 +414,8 @@ private void onMatchNoDocs(SearchRewriteContext rewriteContext, ActionListener<S
         listener.onResponse(QuerySearchResult.nullInstance());
     }
 
-    private <T> void runAsync(long id, Supplier<T> executable, ActionListener<T> listener) {
-        getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
+    private <T> void runAsync(SearchContextId contextId, Supplier<T> executable, ActionListener<T> listener) {
+        getExecutor(contextId).execute(ActionRunnable.supply(listener, executable::get));
     }
 
     private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext, SearchShardTask task) throws Exception {
@@ -468,8 +470,8 @@ private QueryFetchSearchResult executeFetchPhase(SearchContext context, long aft
     public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
-        runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
+        runAsync(request.contextId(), () -> {
+            final SearchContext context = findContext(request.contextId(), request);
             context.incRef();
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 context.setTask(task);
@@ -490,8 +492,8 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
-        runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
+        runAsync(request.contextId(), () -> {
+            final SearchContext context = findContext(request.contextId(), request);
             context.setTask(task);
             context.incRef();
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
@@ -526,10 +528,11 @@ private boolean fetchPhaseShouldFreeContext(SearchContext context) {
         }
     }
 
-    final Executor getExecutor(long id) {
-        SearchContext context = activeContexts.get(id);
+
+    final Executor getExecutor(SearchContextId contextId) {
+        SearchContext context = getContext(contextId);
         if (context == null) {
-            throw new SearchContextMissingException(id);
+            throw new SearchContextMissingException(contextId);
         }
         return getExecutor(context.indexShard());
     }
@@ -541,8 +544,8 @@ private Executor getExecutor(IndexShard indexShard) {
 
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
-        runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
+        runAsync(request.contextId(), () -> {
+            final SearchContext context = findContext(request.contextId(), request);
             context.setTask(task);
             context.incRef();
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)){
@@ -563,8 +566,8 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
-        runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
+        runAsync(request.contextId(), () -> {
+            final SearchContext context = findContext(request.contextId(), request);
             context.incRef();
             try {
                 context.setTask(task);
@@ -576,7 +579,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
                     fetchPhase.execute(context);
                     if (fetchPhaseShouldFreeContext(context)) {
-                        freeContext(request.id());
+                        freeContext(request.contextId());
                     } else {
                         contextProcessedSuccessfully(context);
                     }
@@ -593,10 +596,21 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
         }, listener);
     }
 
-    private SearchContext findContext(long id, TransportRequest request) throws SearchContextMissingException {
-        SearchContext context = activeContexts.get(id);
+    private SearchContext getContext(SearchContextId contextId) {
+        final SearchContext context = activeContexts.get(contextId.getId());
+        if (context == null) {
+            return null;
+        }
+        if (context.id().getReaderId().equals(contextId.getReaderId()) || contextId.getReaderId().isEmpty()) {
+            return context;
+        }
+        return null;
+    }
+
+    private SearchContext findContext(SearchContextId contextId, TransportRequest request) throws SearchContextMissingException {
+        final SearchContext context = getContext(contextId);
         if (context == null) {
-            throw new SearchContextMissingException(id);
+            throw new SearchContextMissingException(contextId);
         }
 
         SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
@@ -628,7 +642,6 @@ private void onNewContext(SearchContext context) {
         boolean success = false;
         try {
             if (context.scrollContext() != null) {
-                openScrollContexts.incrementAndGet();
                 context.indexShard().getSearchOperationListener().onNewScrollContext(context);
             }
             context.indexShard().getSearchOperationListener().onNewContext(context);
@@ -647,14 +660,15 @@ private void onNewContext(SearchContext context) {
     final SearchContext createContext(SearchRewriteContext rewriteContext) throws IOException {
         final DefaultSearchContext context = createSearchContext(rewriteContext, defaultSearchTimeout);
         try {
-            if (rewriteContext.request != null && openScrollContexts.get() >= maxOpenScrollContext) {
-                throw new ElasticsearchException(
-                    "Trying to create too many scroll contexts. Must be less than or equal to: [" +
-                        maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
-                        + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
-            }
             final ShardSearchRequest request = rewriteContext.request;
             if (request.scroll() != null) {
+                context.addReleasable(openScrollContexts::decrementAndGet, Lifetime.CONTEXT);
+                if (openScrollContexts.incrementAndGet() > maxOpenScrollContext) {
+                    throw new ElasticsearchException(
+                        "Trying to create too many scroll contexts. Must be less than or equal to: [" +
+                            maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
+                            + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
+                }
                 context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
             }
@@ -704,8 +718,17 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon
             IndexShard indexShard = indexService.getShard(request.shardId().getId());
             SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
                 indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
-            DefaultSearchContext searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget,
-                searcher, clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
+            // TODO: If no changes are made since the last commit, and the searcher is opened from that commit, then we can use the
+            //  commit_id as the context_id. And if the local checkpoint and max_seq_no of that commit equal the global checkpoint,
+            //  then we can use a combination of history_uuid and one of these values as a **weaker** context_id.
+            //  Reader contexts with the same commit_id can be replaced at any time, as the Lucene doc ids are the same.
+            //  Reader contexts with the same seq_id, however, can't be replaced between the query and fetch phase because
+            //  the Lucene doc ids can be different.
+            final String readerId = UUIDs.base64UUID();
+            DefaultSearchContext searchContext = new DefaultSearchContext(
+                new SearchContextId(readerId, idGenerator.incrementAndGet()),
+                request, shardTarget, searcher, clusterService, indexService, indexShard, bigArrays,
+                threadPool::relativeTimeInMillis, timeout, fetchPhase);
             success = true;
             return searchContext;
         } finally {
@@ -718,6 +741,7 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon
         }
     }
 
+
     private void freeAllContextForIndex(Index index) {
         assert index != null;
         for (SearchContext ctx : activeContexts.values()) {
@@ -727,23 +751,23 @@ private void freeAllContextForIndex(Index index) {
         }
     }
 
-
-    public boolean freeContext(long id) {
-        try (SearchContext context = removeContext(id)) {
-            if (context != null) {
-                onFreeContext(context);
-                return true;
+    public boolean freeContext(SearchContextId contextId) {
+        if (getContext(contextId) != null) {
+            try (SearchContext context = removeContext(contextId.getId())) {
+                if (context != null) {
+                    onFreeContext(context);
+                    return true;
+                }
             }
-            return false;
         }
+        return false;
     }
 
     private void onFreeContext(SearchContext context) {
         assert context.refCount() > 0 : " refCount must be > 0: " + context.refCount();
-        assert activeContexts.containsKey(context.id()) == false;
+        assert activeContexts.containsKey(context.id().getId()) == false;
         context.indexShard().getSearchOperationListener().onFreeContext(context);
         if (context.scrollContext() != null) {
-            openScrollContexts.decrementAndGet();
             context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index 021d6872e3325..77833c98cae0e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -34,6 +34,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 import java.util.function.IntConsumer;
 
 /**
@@ -127,6 +128,29 @@ public String getName() {
         return name;
     }
 
+    /**
+     * Rewrite the sub-aggregations in the buckets in this aggregation.
+     * Returns a copy of this {@linkplain InternalAggregation} with the
+     * rewritten buckets, or, if there aren't any modifications to
+     * the buckets then this method will return this aggregation. Either
+     * way, it doesn't modify this aggregation.
+     * <p>
+     * Implementers of this should call the {@code rewriter} once per bucket
+     * with its {@linkplain InternalAggregations}. The {@code rewriter}
+     * should return {@code null} if it doen't have any rewriting to do or
+     * it should return a new {@linkplain InternalAggregations} to make
+     * changs.
+     * <p>
+     * The default implementation throws an exception because most
+     * aggregations don't <strong>have</strong> buckets in them. It
+     * should be overridden by aggregations that contain buckets. Implementers
+     * should respect the description above.
+     */
+    public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations, InternalAggregations> rewriter) {
+        throw new IllegalStateException(
+                "Aggregation [" + getName() + "] must be a bucket aggregation but was [" + getWriteableName() + "]");
+    }
+
     /**
      * Creates the output from all pipeline aggs that this aggregation is associated with.  Should only
      * be called after all aggregations have been fully reduced
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index 3f18d34fd01b4..07a992d197cd1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -85,6 +85,15 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeNamedWriteableList(topLevelPipelineAggregators);
     }
 
+    /**
+     * Make a mutable copy of the aggregation results.
+     * <p>
+     * IMPORTANT: The copy doesn't include any pipeline aggregations, if there are any.
+     */
+    public List<InternalAggregation> copyResults() {
+        return new ArrayList<>(getInternalAggregations());
+    }
+
     /**
      * Returns the top-level pipeline aggregators.
      * Note that top-level pipeline aggregators become normal aggregation once the final reduction has been performed, after which they
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index 9600c6ae86611..61176fbf29f82 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -29,6 +29,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 public abstract class InternalMultiBucketAggregation<A extends InternalMultiBucketAggregation,
             B extends InternalMultiBucketAggregation.InternalBucket>
@@ -154,6 +155,23 @@ public final InternalAggregation reducePipelines(InternalAggregation reducedAggs
         return super.reducePipelines(create(materializedBuckets), reduceContext);
     }
 
+    @Override
+    public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations, InternalAggregations> rewriter) {
+        boolean modified = false;
+        List<B> newBuckets = new ArrayList<>();
+        for (B bucket : getBuckets()) {
+            InternalAggregations rewritten = rewriter.apply((InternalAggregations) bucket.getAggregations());
+            if (rewritten == null) {
+                newBuckets.add(bucket);
+                continue;
+            }
+            modified = true;
+            B newBucket = createBucket(rewritten, bucket);
+            newBuckets.add(newBucket);
+        }
+        return modified ? create(newBuckets) : this;
+    }
+
     private List<B> reducePipelineBuckets(ReduceContext reduceContext) {
         List<B> reducedBuckets = new ArrayList<>();
         for (B bucket : getBuckets()) {
@@ -192,6 +210,5 @@ public Object getProperty(String containingAggName, List<String> path) {
             }
             return aggregation.getProperty(path.subList(1, path.size()));
         }
-
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
index 68a66318b4511..437cd1466d688 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
@@ -33,6 +33,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 
 /**
  * A base class for all the single bucket aggregations.
@@ -169,6 +170,15 @@ public final double sortValue(AggregationPath.PathElement head, Iterator<Aggrega
         return aggregations.sortValue(head, tail);
     }
 
+    @Override
+    public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations, InternalAggregations> rewriter) {
+        InternalAggregations rewritten = rewriter.apply(aggregations);
+        if (rewritten == null) {
+            return this;
+        }
+        return create(rewritten);
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (this == obj) return true;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
index 78181e3a3366a..801438bc84f7f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
@@ -84,7 +83,7 @@ public long getDocCount() {
         }
 
         @Override
-        public Aggregations getAggregations() {
+        public InternalAggregations getAggregations() {
             return aggregations;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
index 0e01615492939..b4c72b8529c0f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -202,7 +202,8 @@ private Sort buildIndexSortPrefix(LeafReaderContext context) throws IOException
             return null;
         }
         List<SortField> sortFields = new ArrayList<>();
-        for (int i = 0; i < indexSort.getSort().length; i++) {
+        int end = Math.min(indexSort.getSort().length, sourceConfigs.length);
+        for (int i = 0; i < end; i++) {
             CompositeValuesSourceConfig sourceConfig = sourceConfigs[i];
             SingleDimensionValuesSource<?> source = sources[i];
             SortField indexSortField = indexSort.getSort()[i];
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
index d71ed3c3bd97d..93dcdf7be3bda 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
@@ -269,7 +269,8 @@ SortedDocsProducer createSortedDocsProducerOrNull(IndexReader reader, Query quer
             }
             return new PointsSortedDocsProducer(fieldType.name(), toBucketFunction, lowerPoint, upperPoint);
         } else if (fieldType instanceof DateFieldMapper.DateFieldType) {
-            final ToLongFunction<byte[]> toBucketFunction = (value) -> rounding.applyAsLong(LongPoint.decodeDimension(value, 0));
+            ToLongFunction<byte[]> decode = ((DateFieldMapper.DateFieldType) fieldType).resolution()::parsePointAsMillis;
+            ToLongFunction<byte[]> toBucketFunction = value -> rounding.applyAsLong(decode.applyAsLong(value));
             return new PointsSortedDocsProducer(fieldType.name(), toBucketFunction, lowerPoint, upperPoint);
         } else {
             return null;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index d8f74b1b8e42f..ba786f9414778 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -255,31 +255,49 @@ public InternalAggregation reduce(List<InternalAggregation> aggregations, Reduce
             }
         }
 
-        final int size = reduceContext.isFinalReduce() == false ? buckets.size() : Math.min(requiredSize, buckets.size());
-        final BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator());
-        for (List<B> sameTermBuckets : buckets.values()) {
-            final B b = reduceBucket(sameTermBuckets, reduceContext);
-            if (sumDocCountError == -1) {
-                b.docCountError = -1;
-            } else {
-                b.docCountError += sumDocCountError;
+        final B[] list;
+        if (reduceContext.isFinalReduce()) {
+            final int size = Math.min(requiredSize, buckets.size());
+            final BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator());
+            for (List<B> sameTermBuckets : buckets.values()) {
+                final B b = reduceBucket(sameTermBuckets, reduceContext);
+                if (sumDocCountError == -1) {
+                    b.docCountError = -1;
+                } else {
+                    b.docCountError += sumDocCountError;
+                }
+                if (b.docCount >= minDocCount) {
+                    B removed = ordered.insertWithOverflow(b);
+                    if (removed != null) {
+                        otherDocCount += removed.getDocCount();
+                        reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
+                    } else {
+                        reduceContext.consumeBucketsAndMaybeBreak(1);
+                    }
+                } else {
+                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(b));
+                }
             }
-            if (b.docCount >= minDocCount || reduceContext.isFinalReduce() == false) {
-                B removed = ordered.insertWithOverflow(b);
-                if (removed != null) {
-                    otherDocCount += removed.getDocCount();
-                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
+            list = createBucketsArray(ordered.size());
+            for (int i = ordered.size() - 1; i >= 0; i--) {
+                list[i] = ordered.pop();
+            }
+        } else {
+            // keep all buckets on partial reduce
+            // TODO: we could prune the buckets when sorting by key
+            list = createBucketsArray(buckets.size());
+            int pos = 0;
+            for (List<B> sameTermBuckets : buckets.values()) {
+                final B b = reduceBucket(sameTermBuckets, reduceContext);
+                reduceContext.consumeBucketsAndMaybeBreak(1);
+                if (sumDocCountError == -1) {
+                    b.docCountError = -1;
                 } else {
-                    reduceContext.consumeBucketsAndMaybeBreak(1);
+                    b.docCountError += sumDocCountError;
                 }
-            } else {
-                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(b));
+                list[pos++] = b;
             }
         }
-        B[] list = createBucketsArray(ordered.size());
-        for (int i = ordered.size() - 1; i >= 0; i--) {
-            list[i] = ordered.pop();
-        }
         long docCountError;
         if (sumDocCountError == -1) {
             docCountError = -1;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java
index 96563d917bf8f..b8bfae41b2314 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java
@@ -18,15 +18,14 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.util.Bits;
 import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.DoubleArray;
@@ -191,11 +190,7 @@ static Function<byte[], Number> getPointReaderOrNull(SearchContext context, Aggr
                 converter = ((NumberFieldMapper.NumberFieldType) fieldType)::parsePoint;
             } else if (fieldType.getClass() == DateFieldMapper.DateFieldType.class) {
                 DateFieldMapper.DateFieldType dft = (DateFieldMapper.DateFieldType) fieldType;
-                /*
-                 * Makes sure that nanoseconds decode to milliseconds, just
-                 * like they do when you run the agg without the optimization.
-                 */
-                converter = (in) -> dft.resolution().toInstant(LongPoint.decodeDimension(in, 0)).toEpochMilli();
+                converter = dft.resolution()::parsePointAsMillis;
             }
             return converter;
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
index a131dde9c7673..ba7c49cee7eb4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
@@ -24,16 +24,10 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 
 public abstract class SiblingPipelineAggregator extends PipelineAggregator {
     protected SiblingPipelineAggregator(String name, String[] bucketsPaths, Map<String, Object> metaData) {
@@ -47,39 +41,13 @@ protected SiblingPipelineAggregator(String name, String[] bucketsPaths, Map<Stri
         super(in);
     }
 
-    @SuppressWarnings("unchecked")
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
-        if (aggregation instanceof InternalMultiBucketAggregation) {
-            @SuppressWarnings("rawtypes")
-            InternalMultiBucketAggregation multiBucketsAgg = (InternalMultiBucketAggregation) aggregation;
-            List<? extends Bucket> buckets = multiBucketsAgg.getBuckets();
-            List<Bucket> newBuckets = new ArrayList<>();
-            for (Bucket bucket1 : buckets) {
-                InternalMultiBucketAggregation.InternalBucket bucket = (InternalMultiBucketAggregation.InternalBucket) bucket1;
-                InternalAggregation aggToAdd = doReduce(bucket.getAggregations(), reduceContext);
-                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false)
-                    .map((p) -> (InternalAggregation) p)
-                    .collect(Collectors.toList());
-                aggs.add(aggToAdd);
-                InternalMultiBucketAggregation.InternalBucket newBucket = multiBucketsAgg.createBucket(new InternalAggregations(aggs),
-                    bucket);
-                newBuckets.add(newBucket);
-            }
-
-            return multiBucketsAgg.create(newBuckets);
-        } else if (aggregation instanceof InternalSingleBucketAggregation) {
-            InternalSingleBucketAggregation singleBucketAgg = (InternalSingleBucketAggregation) aggregation;
-            InternalAggregation aggToAdd = doReduce(singleBucketAgg.getAggregations(), reduceContext);
-            List<InternalAggregation> aggs = StreamSupport.stream(singleBucketAgg.getAggregations().spliterator(), false)
-                .map((p) -> (InternalAggregation) p)
-                .collect(Collectors.toList());
-            aggs.add(aggToAdd);
-            return singleBucketAgg.create(new InternalAggregations(aggs));
-        } else {
-            throw new IllegalStateException("Aggregation [" + aggregation.getName() + "] must be a bucket aggregation ["
-                    + aggregation.getWriteableName() + "]");
-        }
+        return aggregation.copyWithRewritenBuckets(aggregations -> {
+            List<InternalAggregation> aggs = aggregations.copyResults();
+            aggs.add(doReduce(aggregations, reduceContext));
+            return new InternalAggregations(aggs);
+        });
     }
 
     public abstract InternalAggregation doReduce(Aggregations aggregations, ReduceContext context);
diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
index 4d245e2bcb002..5f931d661674e 100644
--- a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.SearchContextId;
 
 import java.io.IOException;
 
@@ -44,7 +45,7 @@ public class DfsSearchResult extends SearchPhaseResult {
 
     public DfsSearchResult(StreamInput in) throws IOException {
         super(in);
-        requestId = in.readLong();
+        contextId = new SearchContextId(in);
         int termsSize = in.readVInt();
         if (termsSize == 0) {
             terms = EMPTY_TERMS;
@@ -60,9 +61,9 @@ public DfsSearchResult(StreamInput in) throws IOException {
         maxDoc = in.readVInt();
     }
 
-    public DfsSearchResult(long id, SearchShardTarget shardTarget) {
+    public DfsSearchResult(SearchContextId contextId, SearchShardTarget shardTarget) {
         this.setSearchShardTarget(shardTarget);
-        this.requestId = id;
+        this.contextId = contextId;
     }
 
     public DfsSearchResult maxDoc(int maxDoc) {
@@ -99,7 +100,7 @@ public ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics() {
 
   @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeLong(requestId);
+        contextId.writeTo(out);
         out.writeVInt(terms.length);
         for (Term term : terms) {
             out.writeString(term.field());
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
index b213d2bc19d5a..81a2f5491a9a3 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
@@ -40,12 +41,12 @@ public FetchSearchResult() {
 
     public FetchSearchResult(StreamInput in) throws IOException {
         super(in);
-        requestId = in.readLong();
+        contextId = new SearchContextId(in);
         hits = new SearchHits(in);
     }
 
-    public FetchSearchResult(long id, SearchShardTarget shardTarget) {
-        this.requestId = id;
+    public FetchSearchResult(SearchContextId id, SearchShardTarget shardTarget) {
+        this.contextId = id;
         setSearchShardTarget(shardTarget);
     }
 
@@ -86,7 +87,7 @@ public int counterGetAndIncrement() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeLong(requestId);
+        contextId.writeTo(out);
         hits.writeTo(out);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
index 754735c0cd224..dc76fdf46f792 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
@@ -44,8 +45,8 @@ public QueryFetchSearchResult(QuerySearchResult queryResult, FetchSearchResult f
     }
 
     @Override
-    public long getRequestId() {
-        return queryResult.getRequestId();
+    public SearchContextId getContextId() {
+        return queryResult.getContextId();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
index 2e7c59b329874..528d584dfe044 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -39,7 +40,7 @@
  */
 public class ShardFetchRequest extends TransportRequest {
 
-    private long id;
+    private SearchContextId contextId;
 
     private int[] docIds;
 
@@ -50,8 +51,8 @@ public class ShardFetchRequest extends TransportRequest {
     public ShardFetchRequest() {
     }
 
-    public ShardFetchRequest(long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
-        this.id = id;
+    public ShardFetchRequest(SearchContextId contextId, IntArrayList list, ScoreDoc lastEmittedDoc) {
+        this.contextId = contextId;
         this.docIds = list.buffer;
         this.size = list.size();
         this.lastEmittedDoc = lastEmittedDoc;
@@ -59,7 +60,7 @@ public ShardFetchRequest(long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
 
     public ShardFetchRequest(StreamInput in) throws IOException {
         super(in);
-        id = in.readLong();
+        contextId = new SearchContextId(in);
         size = in.readVInt();
         docIds = new int[size];
         for (int i = 0; i < size; i++) {
@@ -78,7 +79,7 @@ public ShardFetchRequest(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeLong(id);
+        contextId.writeTo(out);
         out.writeVInt(size);
         for (int i = 0; i < size; i++) {
             out.writeVInt(docIds[i]);
@@ -94,8 +95,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    public long id() {
-        return id;
+    public SearchContextId contextId() {
+        return contextId;
     }
 
     public int[] docIds() {
@@ -117,7 +118,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
 
     @Override
     public String getDescription() {
-        return "id[" + id + "], size[" + size + "], lastEmittedDoc[" + lastEmittedDoc + "]";
+        return "id[" + contextId + "], size[" + size + "], lastEmittedDoc[" + lastEmittedDoc + "]";
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index 647d629a390c6..813ffc9483375 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.internal.SearchContextId;
 
 import java.io.IOException;
 
@@ -41,7 +42,7 @@ public ShardFetchSearchRequest() {
 
     }
 
-    public ShardFetchSearchRequest(OriginalIndices originalIndices, long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
+    public ShardFetchSearchRequest(OriginalIndices originalIndices, SearchContextId id, IntArrayList list, ScoreDoc lastEmittedDoc) {
         super(id, list, lastEmittedDoc);
         this.originalIndices = originalIndices;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
index b792be60f78fc..d1743e6182e2f 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
@@ -102,6 +103,12 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         ArrayList<TextFragment> fragsList = new ArrayList<>();
         List<Object> textsToHighlight;
         Analyzer analyzer = context.getMapperService().documentMapper().mappers().indexAnalyzer();
+        Integer keywordIgnoreAbove = null;
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+            KeywordFieldMapper mapper = (KeywordFieldMapper) context.getMapperService().documentMapper()
+                .mappers().getMapper(highlighterContext.fieldName);
+            keywordIgnoreAbove = mapper.ignoreAbove();
+        }
         final int maxAnalyzedOffset = context.getIndexSettings().getHighlightMaxAnalyzedOffset();
 
         try {
@@ -110,7 +117,11 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
 
             for (Object textToHighlight : textsToHighlight) {
                 String text = convertFieldValue(fieldType, textToHighlight);
-                if (text.length() > maxAnalyzedOffset) {
+                int textLength = text.length();
+                if (keywordIgnoreAbove != null  && textLength > keywordIgnoreAbove) {
+                    continue; // skip highlighting keyword terms that were ignored during indexing
+                }
+                if (textLength > maxAnalyzedOffset) {
                     throw new IllegalArgumentException(
                         "The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +
                             "] doc of [" + context.index().getName() + "] index " +
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
index 81f700920f827..28ff7220d9bd7 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
@@ -65,11 +66,16 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
         Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
         final int maxAnalyzedOffset = context.getIndexSettings().getHighlightMaxAnalyzedOffset();
+        Integer keywordIgnoreAbove = null;
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+            KeywordFieldMapper mapper = (KeywordFieldMapper) context.getMapperService().documentMapper()
+                .mappers().getMapper(highlighterContext.fieldName);
+            keywordIgnoreAbove = mapper.ignoreAbove();
+        }
 
         List<Snippet> snippets = new ArrayList<>();
         int numberOfFragments = field.fieldOptions().numberOfFragments();
         try {
-
             final Analyzer analyzer = getAnalyzer(context.getMapperService().documentMapper(), hitContext);
             List<Object> fieldValues = loadFieldValues(fieldType, field, context, hitContext,
                 highlighterContext.highlight.forceSource(field));
@@ -81,7 +87,11 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
             final CustomUnifiedHighlighter highlighter;
             final String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
             final OffsetSource offsetSource = getOffsetSource(fieldType);
-            if ((offsetSource == OffsetSource.ANALYSIS) && (fieldValue.length() > maxAnalyzedOffset)) {
+            int fieldValueLength = fieldValue.length();
+            if (keywordIgnoreAbove != null  && fieldValueLength > keywordIgnoreAbove) {
+                return null; // skip highlighting keyword terms that were ignored during indexing
+            }
+            if ((offsetSource == OffsetSource.ANALYSIS) && (fieldValueLength > maxAnalyzedOffset)) {
                 throw new IllegalArgumentException(
                     "The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +
                         "] doc of [" + context.index().getName() + "] index " + "has exceeded [" +
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 8c04954a4efcf..41c46fc6a81be 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -106,7 +106,7 @@ public Query buildFilteredQuery(Query query) {
     }
 
     @Override
-    public long id() {
+    public SearchContextId id() {
         return in.id();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
index fe8d173711c69..ff86e44c1704b 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
@@ -33,33 +33,33 @@
 
 public class InternalScrollSearchRequest extends TransportRequest {
 
-    private long id;
+    private SearchContextId contextId;
 
     private Scroll scroll;
 
     public InternalScrollSearchRequest() {
     }
 
-    public InternalScrollSearchRequest(SearchScrollRequest request, long id) {
-        this.id = id;
+    public InternalScrollSearchRequest(SearchScrollRequest request, SearchContextId contextId) {
+        this.contextId = contextId;
         this.scroll = request.scroll();
     }
 
     public InternalScrollSearchRequest(StreamInput in) throws IOException {
         super(in);
-        id = in.readLong();
+        contextId = new SearchContextId(in);
         scroll = in.readOptionalWriteable(Scroll::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeLong(id);
+        contextId.writeTo(out);
         out.writeOptionalWriteable(scroll);
     }
 
-    public long id() {
-        return id;
+    public SearchContextId contextId() {
+        return contextId;
     }
 
     public Scroll scroll() {
@@ -78,7 +78,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
 
     @Override
     public String getDescription() {
-        return "id[" + id + "], scroll[" + scroll + "]";
+        return "id[" + contextId.getId() + "], scroll[" + scroll + "]";
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 4473546f76d15..67577d61a7454 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -132,7 +132,7 @@ protected void alreadyClosed() {
      *  alias filters, types filters, etc. */
     public abstract Query buildFilteredQuery(Query query);
 
-    public abstract long id();
+    public abstract SearchContextId id();
 
     public abstract String source();
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
new file mode 100644
index 0000000000000..38513dcc5b7d3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Objects;
+
+
+public final class SearchContextId implements Writeable {
+    private final String readerId;
+    private final long id;
+
+    public SearchContextId(String readerId, long id) {
+        this.readerId = Objects.requireNonNull(readerId);
+        this.id = id;
+    }
+
+    public SearchContextId(StreamInput in) throws IOException {
+        this.id = in.readLong();
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+            this.readerId = in.readString();
+        } else {
+            this.readerId = "";
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(id);
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+            out.writeString(readerId);
+        }
+    }
+
+    public String getReaderId() {
+        return readerId;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SearchContextId other = (SearchContextId) o;
+        return id == other.id && readerId.equals(other.readerId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(readerId, id);
+    }
+
+    @Override
+    public String toString() {
+        return "[" + readerId + "][" + id + "]";
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
index d3919ec3aba48..d85d6e674c634 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -36,24 +37,21 @@
 
 public class QuerySearchRequest extends TransportRequest implements IndicesRequest {
 
-    private long id;
+    private final SearchContextId contextId;
 
-    private AggregatedDfs dfs;
+    private final AggregatedDfs dfs;
 
-    private OriginalIndices originalIndices;
+    private final OriginalIndices originalIndices;
 
-    public QuerySearchRequest() {
-    }
-
-    public QuerySearchRequest(OriginalIndices originalIndices, long id, AggregatedDfs dfs) {
-        this.id = id;
+    public QuerySearchRequest(OriginalIndices originalIndices, SearchContextId contextId, AggregatedDfs dfs) {
+        this.contextId = contextId;
         this.dfs = dfs;
         this.originalIndices = originalIndices;
     }
 
     public QuerySearchRequest(StreamInput in) throws IOException {
         super(in);
-        id = in.readLong();
+        contextId = new SearchContextId(in);
         dfs = new AggregatedDfs(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
     }
@@ -61,13 +59,13 @@ public QuerySearchRequest(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeLong(id);
+        contextId.writeTo(out);
         dfs.writeTo(out);
         OriginalIndices.writeOriginalIndices(originalIndices, out);
     }
 
-    public long id() {
-        return id;
+    public SearchContextId contextId() {
+        return contextId;
     }
 
     public AggregatedDfs dfs() {
@@ -92,7 +90,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
     public String getDescription() {
         StringBuilder sb = new StringBuilder();
         sb.append("id[");
-        sb.append(id);
+        sb.append(contextId);
         sb.append("], ");
         sb.append("indices[");
         Strings.arrayToDelimitedString(originalIndices.indices(), ",", sb);
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 8c62d229fe766..18af04add2082 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 
@@ -77,13 +78,13 @@ public QuerySearchResult(StreamInput in) throws IOException {
             isNull = false;
         }
         if (isNull == false) {
-            long id = in.readLong();
+            SearchContextId id = new SearchContextId(in);
             readFromWithId(id, in);
         }
     }
 
-    public QuerySearchResult(long id, SearchShardTarget shardTarget) {
-        this.requestId = id;
+    public QuerySearchResult(SearchContextId id, SearchShardTarget shardTarget) {
+        this.contextId = id;
         setSearchShardTarget(shardTarget);
         isNull = false;
     }
@@ -298,8 +299,8 @@ public boolean hasSearchContext() {
         return hasScoreDocs || hasSuggestHits();
     }
 
-    public void readFromWithId(long id, StreamInput in) throws IOException {
-        this.requestId = id;
+    public void readFromWithId(SearchContextId id, StreamInput in) throws IOException {
+        this.contextId = id;
         from = in.readVInt();
         size = in.readVInt();
         int numSortFieldsPlus1 = in.readVInt();
@@ -344,7 +345,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(isNull);
         }
         if (isNull == false) {
-            out.writeLong(requestId);
+            contextId.writeTo(out);
             writeToNoId(out);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
index ab02d6df7993b..703ae6939aa8f 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
@@ -55,14 +55,14 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Return the minimum value.
      */
-    T getMin() {
+    public T getMin() {
         return minValue;
     }
 
     /**
      * Return the maximum value.
      */
-    T getMax() {
+    public T getMax() {
         return maxValue;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortValue.java b/server/src/main/java/org/elasticsearch/search/sort/SortValue.java
index f4cfb812b7a6d..92fbe3cfc4875 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/SortValue.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/SortValue.java
@@ -117,6 +117,11 @@ public final XContentBuilder toXContent(XContentBuilder builder, DocValueFormat
     @Override
     public abstract String toString();
 
+    /**
+     * Return this {@linkplain SortValue} as a boxed {@linkplain Number}.
+     */
+    public abstract Number numberValue();
+
     private static class DoubleSortValue extends SortValue {
         public static final String NAME = "double";
 
@@ -179,6 +184,11 @@ public int hashCode() {
         public String toString() {
             return Double.toString(key);
         }
+
+        @Override
+        public Number numberValue() {
+            return key;
+        }
     }
 
     private static class LongSortValue extends SortValue {
@@ -243,5 +253,10 @@ public int hashCode() {
         public String toString() {
             return Long.toString(key);
         }
+
+        @Override
+        public Number numberValue() {
+            return key;
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/tasks/Task.java b/server/src/main/java/org/elasticsearch/tasks/Task.java
index 1f89a7d88b993..0c821d030e624 100644
--- a/server/src/main/java/org/elasticsearch/tasks/Task.java
+++ b/server/src/main/java/org/elasticsearch/tasks/Task.java
@@ -140,6 +140,13 @@ public long getStartTime() {
         return startTime;
     }
 
+    /**
+     * Returns the task's start time in nanoseconds ({@link System#nanoTime()} style).
+     */
+    public long getStartTimeNanos() {
+        return startTimeNanos;
+    }
+
     /**
      * Returns id of the parent task or NO_PARENT_ID if the task doesn't have any parent tasks
      */
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index fe4944fce06a8..5a2b83c67222d 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -68,7 +68,6 @@ public class ThreadPool implements Scheduler {
     public static class Names {
         public static final String SAME = "same";
         public static final String GENERIC = "generic";
-        public static final String LISTENER = "listener";
         public static final String GET = "get";
         public static final String ANALYZE = "analyze";
         public static final String WRITE = "write";
@@ -115,7 +114,6 @@ public static ThreadPoolType fromType(String type) {
     public static final Map<String, ThreadPoolType> THREAD_POOL_TYPES = Map.ofEntries(
         entry(Names.SAME, ThreadPoolType.DIRECT),
         entry(Names.GENERIC, ThreadPoolType.SCALING),
-        entry(Names.LISTENER, ThreadPoolType.FIXED),
         entry(Names.GET, ThreadPoolType.FIXED),
         entry(Names.ANALYZE, ThreadPoolType.FIXED),
         entry(Names.WRITE, ThreadPoolType.FIXED),
@@ -172,7 +170,6 @@ public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBui
         builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5)));
         // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded
         // the assumption here is that the listeners should be very lightweight on the listeners side
-        builders.put(Names.LISTENER, new FixedExecutorBuilder(settings, Names.LISTENER, halfProcMaxAt10, -1, false));
         builders.put(Names.FLUSH, new ScalingExecutorBuilder(Names.FLUSH, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
         builders.put(Names.REFRESH, new ScalingExecutorBuilder(Names.REFRESH, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5)));
         builders.put(Names.WARMER, new ScalingExecutorBuilder(Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index 5ce6de7998801..2d314a591b57a 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -272,9 +272,9 @@ private ProxyModeInfo(StreamInput input) throws IOException {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field("address", address);
-            builder.field("num_sockets_connected", numSocketsConnected);
-            builder.field("max_socket_connections", maxSocketConnections);
+            builder.field("proxy_address", address);
+            builder.field("num_proxy_sockets_connected", numSocketsConnected);
+            builder.field("max_proxy_socket_connections", maxSocketConnections);
             return builder;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
index 52c1bd7a24d24..0321f247b0843 100644
--- a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
+++ b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
@@ -56,6 +57,7 @@
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.RemoteTransportException;
 
@@ -818,7 +820,8 @@ public void testFailureToAndFromXContentWithDetails() throws IOException {
                                 new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 61), null,
                                     OriginalIndices.NONE)), new ShardSearchFailure(new RepositoryException("repository_g", "Repo"),
                                 new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62), null,
-                                    OriginalIndices.NONE)), new ShardSearchFailure(new SearchContextMissingException(0L), null)
+                                    OriginalIndices.NONE)), new ShardSearchFailure(
+                                        new SearchContextMissingException(new SearchContextId(UUIDs.randomBase64UUID(), 0L)), null)
                 };
                 failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
 
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index cad156555417e..f2a73573ee1b7 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -79,6 +79,7 @@
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -120,6 +121,7 @@
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.test.TestSearchContext.SHARD_TARGET;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class ExceptionSerializationTests extends ESTestCase {
@@ -351,9 +353,15 @@ public void testActionTransportException() throws IOException {
     }
 
     public void testSearchContextMissingException() throws IOException {
-        long id = randomLong();
-        SearchContextMissingException ex = serialize(new SearchContextMissingException(id));
-        assertEquals(id, ex.id());
+        SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomLong());
+        Version version = VersionUtils.randomVersion(random());
+        SearchContextMissingException ex = serialize(new SearchContextMissingException(contextId), version);
+        assertThat(ex.contextId().getId(), equalTo(contextId.getId()));
+        if (version.onOrAfter(Version.V_7_7_0)) {
+            assertThat(ex.contextId().getReaderId(), equalTo(contextId.getReaderId()));
+        } else {
+            assertThat(ex.contextId().getReaderId(), equalTo(""));
+        }
     }
 
     public void testCircuitBreakingException() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java
new file mode 100644
index 0000000000000..edf334155238b
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.stats;
+
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class NodesStatsRequestTests extends ESTestCase {
+
+    /**
+     * Make sure that we can set, serialize, and deserialize arbitrary sets
+     * of metrics.
+     *
+     * TODO: Use a looping construct rather than direct API calls
+     */
+    public void testMetricsSetters() throws Exception {
+        NodesStatsRequest request = new NodesStatsRequest(randomAlphaOfLength(8));
+        request.indices(randomFrom(CommonStatsFlags.ALL));
+        request.os(randomBoolean());
+        request.process(randomBoolean());
+        request.jvm(randomBoolean());
+        request.threadPool(randomBoolean());
+        request.fs(randomBoolean());
+        request.transport(randomBoolean());
+        request.http(randomBoolean());
+        request.breaker(randomBoolean());
+        request.script(randomBoolean());
+        request.discovery(randomBoolean());
+        request.ingest(randomBoolean());
+        request.adaptiveSelection(randomBoolean());
+        NodesStatsRequest deserializedRequest = roundTripRequest(request);
+        assertRequestsEqual(request, deserializedRequest);
+    }
+
+    /**
+     * Test that a newly constructed NodesStatsRequestObject requests all of the
+     * possible metrics defined in {@link NodesStatsRequest}.
+     */
+    public void testNodesStatsRequestDefaults() {
+        NodesStatsRequest defaultNodesStatsRequest = new NodesStatsRequest(randomAlphaOfLength(8));
+        NodesStatsRequest constructedNodesStatsRequest = new NodesStatsRequest(randomAlphaOfLength(8));
+        constructedNodesStatsRequest.clear();
+        constructedNodesStatsRequest.indices(CommonStatsFlags.ALL);
+
+        assertRequestsEqual(defaultNodesStatsRequest, constructedNodesStatsRequest);
+    }
+
+    /**
+     * Test that the {@link NodesStatsRequest#all()} method sets all of the
+     * metrics to {@code true}.
+     *
+     * TODO: Use a looping construct rather than direct API calls
+     */
+    public void testNodesInfoRequestAll() throws Exception {
+        NodesStatsRequest request = new NodesStatsRequest("node");
+        request.all();
+
+        assertThat(request.indices().getFlags(), equalTo(CommonStatsFlags.ALL.getFlags()));
+        assertTrue(request.os());
+        assertTrue(request.process());
+        assertTrue(request.jvm());
+        assertTrue(request.threadPool());
+        assertTrue(request.fs());
+        assertTrue(request.transport());
+        assertTrue(request.http());
+        assertTrue(request.breaker());
+        assertTrue(request.script());
+        assertTrue(request.discovery());
+        assertTrue(request.ingest());
+        assertTrue(request.adaptiveSelection());
+    }
+
+    /**
+     * Test that the {@link NodesStatsRequest#clear()} method sets all of the
+     * metrics to {@code false}.
+     *
+     * TODO: Use a looping construct rather than direct API calls
+     */
+    public void testNodesInfoRequestClear() throws Exception {
+        NodesStatsRequest request = new NodesStatsRequest("node");
+        request.clear();
+
+        assertThat(request.indices().getFlags(), equalTo(CommonStatsFlags.NONE.getFlags()));
+        assertFalse(request.os());
+        assertFalse(request.process());
+        assertFalse(request.jvm());
+        assertFalse(request.threadPool());
+        assertFalse(request.fs());
+        assertFalse(request.transport());
+        assertFalse(request.http());
+        assertFalse(request.breaker());
+        assertFalse(request.script());
+        assertFalse(request.discovery());
+        assertFalse(request.ingest());
+        assertFalse(request.adaptiveSelection());
+    }
+
+    /**
+     * Serialize and deserialize a request.
+     * @param request A request to serialize.
+     * @return The deserialized, "round-tripped" request.
+     */
+    private static NodesStatsRequest roundTripRequest(NodesStatsRequest request) throws Exception {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                return new NodesStatsRequest(in);
+            }
+        }
+    }
+
+    private static void assertRequestsEqual(NodesStatsRequest request1, NodesStatsRequest request2) {
+        // TODO: Use a looping construct rather than direct API calls
+        assertThat(request1.indices().getFlags(), equalTo(request2.indices().getFlags()));
+        assertThat(request1.os(), equalTo(request2.os()));
+        assertThat(request1.process(), equalTo(request2.process()));
+        assertThat(request1.jvm(), equalTo(request2.jvm()));
+        assertThat(request1.threadPool(), equalTo(request2.threadPool()));
+        assertThat(request1.fs(), equalTo(request2.fs()));
+        assertThat(request1.transport(), equalTo(request2.transport()));
+        assertThat(request1.http(), equalTo(request2.http()));
+        assertThat(request1.breaker(), equalTo(request2.breaker()));
+        assertThat(request1.script(), equalTo(request2.script()));
+        assertThat(request1.discovery(), equalTo(request2.discovery()));
+        assertThat(request1.ingest(), equalTo(request2.ingest()));
+        assertThat(request1.adaptiveSelection(), equalTo(request2.adaptiveSelection()));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
index 59518b6e9966f..2254c6ba1032f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
@@ -61,16 +61,6 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) {
         instance.partial(randomBoolean());
         instance.includeAliases(randomBoolean());
 
-        if (randomBoolean()) {
-            Map<String, Object> settings = new HashMap<>();
-            int count = randomInt(3) + 1;
-
-            for (int i = 0; i < count; ++i) {
-                settings.put(randomAlphaOfLengthBetween(2, 5), randomAlphaOfLengthBetween(2, 5));
-            }
-
-            instance.settings(settings);
-        }
         if (randomBoolean()) {
             Map<String, Object> indexSettings = new HashMap<>();
             int count = randomInt(3) + 1;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java
new file mode 100644
index 0000000000000..095a166f4f0b7
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.forcemerge;
+
+import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ForceMergeIT extends ESIntegTestCase {
+
+    public void testForceMergeUUIDConsistent() throws IOException {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        final String index = "test-index";
+        createIndex(index,
+            Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build());
+        ensureGreen(index);
+        final ClusterState state = clusterService().state();
+        final IndexRoutingTable indexShardRoutingTables = state.routingTable().getIndicesRouting().get(index);
+        final IndexShardRoutingTable shardRouting = indexShardRoutingTables.getShards().get(0);
+        final String primaryNodeId = shardRouting.primaryShard().currentNodeId();
+        final String replicaNodeId = shardRouting.replicaShards().get(0).currentNodeId();
+        final Index idx = shardRouting.primaryShard().index();
+        final IndicesService primaryIndicesService =
+            internalCluster().getInstance(IndicesService.class, state.nodes().get(primaryNodeId).getName());
+        final IndicesService replicaIndicesService = internalCluster().getInstance(IndicesService.class,
+            state.nodes().get(replicaNodeId).getName());
+        final IndexShard primary = primaryIndicesService.indexService(idx).getShard(0);
+        final IndexShard replica = replicaIndicesService.indexService(idx).getShard(0);
+
+        assertThat(getForceMergeUUID(primary), nullValue());
+        assertThat(getForceMergeUUID(replica), nullValue());
+
+        final ForceMergeResponse forceMergeResponse =
+            client().admin().indices().prepareForceMerge(index).setMaxNumSegments(1).get();
+
+        assertThat(forceMergeResponse.getFailedShards(), is(0));
+        assertThat(forceMergeResponse.getSuccessfulShards(), is(2));
+
+        // Force flush to force a new commit that contains the force flush UUID
+        final FlushResponse flushResponse = client().admin().indices().prepareFlush(index).setForce(true).get();
+        assertThat(flushResponse.getFailedShards(), is(0));
+        assertThat(flushResponse.getSuccessfulShards(), is(2));
+
+        final String primaryForceMergeUUID = getForceMergeUUID(primary);
+        assertThat(primaryForceMergeUUID, notNullValue());
+
+        final String replicaForceMergeUUID = getForceMergeUUID(replica);
+        assertThat(replicaForceMergeUUID, notNullValue());
+        assertThat(primaryForceMergeUUID, is(replicaForceMergeUUID));
+    }
+
+    private static String getForceMergeUUID(IndexShard indexShard) throws IOException {
+        try (Engine.IndexCommitRef indexCommitRef = indexShard.acquireLastIndexCommit(true)) {
+            return indexCommitRef.getIndexCommit().getUserData().get(Engine.FORCE_MERGE_UUID_KEY);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index a01c1f44a73d9..bd45128160e56 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.ingest.CompoundProcessor;
 import org.elasticsearch.ingest.IngestDocument;
@@ -45,6 +46,7 @@
 import static org.elasticsearch.ingest.IngestDocument.MetaData.ROUTING;
 import static org.elasticsearch.ingest.IngestDocument.MetaData.VERSION;
 import static org.elasticsearch.ingest.IngestDocument.MetaData.VERSION_TYPE;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -216,4 +218,33 @@ public void testNonExistentPipelineId() {
             () -> SimulatePipelineRequest.parseWithPipelineId(pipelineId, requestContent, false, ingestService));
         assertThat(e.getMessage(), equalTo("pipeline [" + pipelineId + "] does not exist"));
     }
+
+    public void testNotValidDocs() {
+        Map<String, Object> requestContent = new HashMap<>();
+        List<Map<String, Object>> docs = new ArrayList<>();
+        Map<String, Object> pipelineConfig = new HashMap<>();
+        List<Map<String, Object>> processors = new ArrayList<>();
+        pipelineConfig.put("processors", processors);
+        requestContent.put(Fields.DOCS, docs);
+        requestContent.put(Fields.PIPELINE, pipelineConfig);
+        Exception e1 = expectThrows(IllegalArgumentException.class,
+            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService));
+        assertThat(e1.getMessage(), equalTo("must specify at least one document in [docs]"));
+
+        List<String> stringList = new ArrayList<>();
+        stringList.add("test");
+        pipelineConfig.put("processors", processors);
+        requestContent.put(Fields.DOCS, stringList);
+        requestContent.put(Fields.PIPELINE, pipelineConfig);
+        Exception e2 = expectThrows(IllegalArgumentException.class,
+            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService));
+        assertThat(e2.getMessage(), equalTo("malformed [docs] section, should include an inner object"));
+
+        docs.add(new HashMap<>());
+        requestContent.put(Fields.DOCS, docs);
+        requestContent.put(Fields.PIPELINE, pipelineConfig);
+        Exception e3 = expectThrows(ElasticsearchParseException.class,
+            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService));
+        assertThat(e3.getMessage(), containsString("required property is missing"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
index 2583e1a76c427..b8957b7fef272 100644
--- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -22,8 +22,10 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
@@ -33,6 +35,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
@@ -55,7 +58,7 @@
 public class AbstractSearchAsyncActionTests extends ESTestCase {
 
     private final List<Tuple<String, String>> resolvedNodes = new ArrayList<>();
-    private final Set<Long> releasedContexts = new CopyOnWriteArraySet<>();
+    private final Set<SearchContextId> releasedContexts = new CopyOnWriteArraySet<>();
 
     private AbstractSearchAsyncAction<SearchPhaseResult> createAction(SearchRequest request,
                                                                       ArraySearchPhaseResults<SearchPhaseResult> results,
@@ -90,7 +93,7 @@ private AbstractSearchAsyncAction<SearchPhaseResult> createAction(SearchRequest
                     Collections.singletonList(
                         new SearchShardIterator(null, null, Collections.emptyList(), null)
                     )
-                ), timeProvider, 0, null,
+                ), timeProvider, ClusterState.EMPTY_STATE, null,
                 results, request.getMaxConcurrentShardRequests(),
                 SearchResponse.Clusters.EMPTY) {
             @Override
@@ -110,7 +113,8 @@ long buildTookInMillis() {
             }
 
             @Override
-            public void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+            public void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection,
+                                                 OriginalIndices originalIndices) {
                 releasedContexts.add(contextId);
             }
         };
@@ -191,7 +195,7 @@ public void testSendSearchResponseDisallowPartialFailures() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false);
         AtomicReference<Exception> exception = new AtomicReference<>();
         ActionListener<SearchResponse> listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
-        Set<Long> requestIds = new HashSet<>();
+        Set<SearchContextId> requestIds = new HashSet<>();
         List<Tuple<String, String>> nodeLookups = new ArrayList<>();
         int numFailures = randomIntBetween(1, 5);
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = phaseResults(requestIds, nodeLookups, numFailures);
@@ -219,7 +223,7 @@ public void testOnPhaseFailure() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false);
         AtomicReference<Exception> exception = new AtomicReference<>();
         ActionListener<SearchResponse> listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
-        Set<Long> requestIds = new HashSet<>();
+        Set<SearchContextId> requestIds = new HashSet<>();
         List<Tuple<String, String>> nodeLookups = new ArrayList<>();
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = phaseResults(requestIds, nodeLookups, 0);
         AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest, phaseResults, listener, false, new AtomicLong());
@@ -262,16 +266,16 @@ public void testShardNotAvailableWithDisallowPartialFailures() {
         assertEquals(0, searchPhaseExecutionException.getSuppressed().length);
     }
 
-    private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<Long> requestIds,
-                                                                                              List<Tuple<String, String>> nodeLookups,
-                                                                                              int numFailures) {
+    private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<SearchContextId> contextIds,
+                                                                           List<Tuple<String, String>> nodeLookups,
+                                                                           int numFailures) {
         int numResults = randomIntBetween(1, 10);
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = new ArraySearchPhaseResults<>(numResults + numFailures);
 
         for (int i = 0; i < numResults; i++) {
-            long requestId = randomLong();
-            requestIds.add(requestId);
-            SearchPhaseResult phaseResult = new PhaseResult(requestId);
+            SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomNonNegativeLong());
+            contextIds.add(contextId);
+            SearchPhaseResult phaseResult = new PhaseResult(contextId);
             String resultClusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10);
             String resultNodeId = randomAlphaOfLengthBetween(5, 10);
             ShardId resultShardId = new ShardId("index", "index-uuid", i);
@@ -284,8 +288,8 @@ private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<Long>
     }
 
     private static final class PhaseResult extends SearchPhaseResult {
-        PhaseResult(long requestId) {
-            this.requestId = requestId;
+        PhaseResult(SearchContextId contextId) {
+            this.contextId = contextId;
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
index c38f77207daf8..3d46a3e13a434 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
@@ -18,9 +18,11 @@
  */
 package org.elasticsearch.action.search;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -53,6 +55,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.IntStream;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class CanMatchPreFilterSearchPhaseTests extends ESTestCase {
 
     public void testFilterShards() throws InterruptedException {
@@ -90,7 +94,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req
             (clusterAlias, node) -> lookup.get(node),
             Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
             Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
-            searchRequest, null, shardsIter, timeProvider, 0, null,
+            searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
             (iter) -> new SearchPhase("test") {
                     @Override
                     public void run() throws IOException {
@@ -158,7 +162,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req
             (clusterAlias, node) -> lookup.get(node),
             Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
             Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
-            searchRequest, null, shardsIter, timeProvider, 0, null,
+            searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
             (iter) -> new SearchPhase("test") {
                 @Override
                 public void run() throws IOException {
@@ -225,7 +229,7 @@ public void sendCanMatch(
             null,
             shardsIter,
             timeProvider,
-            0,
+            ClusterState.EMPTY_STATE,
             null,
             (iter) -> new AbstractSearchAsyncAction<>(
                 "test",
@@ -243,7 +247,7 @@ public void sendCanMatch(
                 responseListener,
                 iter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                0,
+                ClusterState.EMPTY_STATE,
                 null,
                 new ArraySearchPhaseResults<>(iter.size()),
                 randomIntBetween(1, 32),
@@ -325,7 +329,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req
                 (clusterAlias, node) -> lookup.get(node),
                 Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
                 Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
-                searchRequest, null, shardsIter, timeProvider, 0, null,
+                searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
                 (iter) -> new SearchPhase("test") {
                     @Override
                     public void run() {
@@ -349,4 +353,76 @@ public void run() {
             }
         }
     }
+
+    public void testInvalidSortShards() throws InterruptedException {
+        final TransportSearchAction.SearchTimeProvider timeProvider =
+            new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), System::nanoTime);
+
+        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
+        lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode));
+        lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode));
+
+        for (SortOrder order : SortOrder.values()) {
+            int numShards = randomIntBetween(2, 20);
+            List<ShardId> shardIds = new ArrayList<>();
+            Set<ShardId> shardToSkip = new HashSet<>();
+
+            SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+                @Override
+                public void sendCanMatch(Transport.Connection connection, ShardSearchRequest request, SearchTask task,
+                                         ActionListener<SearchService.CanMatchResponse> listener) {
+                    final MinAndMax<?> minMax;
+                    if (request.shardId().id() == numShards-1) {
+                        minMax = new MinAndMax<>(new BytesRef("bar"), new BytesRef("baz"));
+                    } else {
+                        Long min = randomLong();
+                        Long max = randomLongBetween(min, Long.MAX_VALUE);
+                        minMax = new MinAndMax<>(min, max);
+                    }
+                    boolean canMatch = frequently();
+                    synchronized (shardIds) {
+                        shardIds.add(request.shardId());
+                        if (canMatch == false) {
+                            shardToSkip.add(request.shardId());
+                        }
+                    }
+                    new Thread(() -> listener.onResponse(new SearchService.CanMatchResponse(canMatch, minMax))).start();
+                }
+            };
+
+            AtomicReference<GroupShardsIterator<SearchShardIterator>> result = new AtomicReference<>();
+            CountDownLatch latch = new CountDownLatch(1);
+            GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("logs",
+                new OriginalIndices(new String[]{"logs"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
+                numShards, randomBoolean(), primaryNode, replicaNode);
+            final SearchRequest searchRequest = new SearchRequest();
+            searchRequest.source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp").order(order)));
+            searchRequest.allowPartialSearchResults(true);
+
+            CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase(logger,
+                searchTransportService,
+                (clusterAlias, node) -> lookup.get(node),
+                Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
+                Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
+                searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
+                (iter) -> new SearchPhase("test") {
+                    @Override
+                    public void run() {
+                        result.set(iter);
+                        latch.countDown();
+                    }
+                }, SearchResponse.Clusters.EMPTY);
+
+            canMatchPhase.start();
+            latch.await();
+            int shardId = 0;
+            for (SearchShardIterator i : result.get()) {
+                assertThat(i.shardId().id(), equalTo(shardId++));
+                assertEquals(shardToSkip.contains(i.shardId()), i.skip());
+            }
+            assertThat(result.get().size(), equalTo(numShards));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
index bcb4a1200b7e8..616b382baf5ca 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
@@ -23,10 +23,12 @@
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.NodeNotConnectedException;
 import org.elasticsearch.transport.Transport;
@@ -92,17 +94,20 @@ public void testClearScrollIds() throws IOException, InterruptedException {
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 1), node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 12), node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 42), node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
         array.setOnce(2, testSearchPhaseResult3);
         AtomicInteger numFreed = new AtomicInteger(0);
-        String scrollId = TransportSearchHelper.buildScrollId(array);
+        String scrollId = TransportSearchHelper.buildScrollId(array, randomBoolean());
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
         CountDownLatch latch = new CountDownLatch(1);
         ActionListener<ClearScrollResponse> listener = new LatchedActionListener<>(new ActionListener<ClearScrollResponse>() {
@@ -121,7 +126,7 @@ public void onFailure(Exception e) {
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
-            public void sendFreeContext(Transport.Connection connection, long contextId,
+            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
                                         ActionListener<SearchFreeContextResponse> listener) {
                 nodesInvoked.add(connection.getNode());
                 boolean freed = randomBoolean();
@@ -153,11 +158,14 @@ public void testClearScrollIdsWithFailure() throws IOException, InterruptedExcep
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 1), node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 12), node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 42), node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
@@ -165,7 +173,7 @@ public void testClearScrollIdsWithFailure() throws IOException, InterruptedExcep
         AtomicInteger numFreed = new AtomicInteger(0);
         AtomicInteger numFailures = new AtomicInteger(0);
         AtomicInteger numConnectionFailures = new AtomicInteger(0);
-        String scrollId = TransportSearchHelper.buildScrollId(array);
+        String scrollId = TransportSearchHelper.buildScrollId(array, randomBoolean());
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
         CountDownLatch latch = new CountDownLatch(1);
 
@@ -189,7 +197,7 @@ public void onFailure(Exception e) {
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
-            public void sendFreeContext(Transport.Connection connection, long contextId,
+            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
                                         ActionListener<SearchFreeContextResponse> listener) {
                 nodesInvoked.add(connection.getNode());
                 boolean freed = randomBoolean();
diff --git a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
index a85f63abd0967..8d20289ca1ae9 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
@@ -19,11 +19,13 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -59,7 +61,7 @@ public void testCollect() throws InterruptedException {
                 case 1:
                     state.add(1);
                     executor.execute(() -> {
-                        DfsSearchResult dfsSearchResult = new DfsSearchResult(shardID, null);
+                        DfsSearchResult dfsSearchResult = new DfsSearchResult(new SearchContextId(UUIDs.randomBase64UUID(), shardID), null);
                         dfsSearchResult.setShardIndex(shardID);
                         dfsSearchResult.setSearchShardTarget(new SearchShardTarget("foo",
                             new ShardId("bar", "baz", shardID), null, OriginalIndices.NONE));
@@ -84,7 +86,7 @@ public void testCollect() throws InterruptedException {
                     break;
                 case 1:
                     assertNotNull(results.get(i));
-                    assertEquals(i, results.get(i).getRequestId());
+                    assertEquals(i, results.get(i).getContextId().getId());
                     break;
                 case 2:
                     final int shardId = i;
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index 8a8e28e15f20d..a2496c19d2a9f 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -34,6 +35,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
@@ -45,8 +47,8 @@
 
 public class DfsQueryPhaseTests extends ESTestCase {
 
-    private static DfsSearchResult newSearchResult(int shardIndex, long requestId, SearchShardTarget target) {
-        DfsSearchResult result = new DfsSearchResult(requestId, target);
+    private static DfsSearchResult newSearchResult(int shardIndex, SearchContextId contextId, SearchShardTarget target) {
+        DfsSearchResult result = new DfsSearchResult(contextId, target);
         result.setShardIndex(shardIndex);
         return result;
     }
@@ -54,8 +56,10 @@ private static DfsSearchResult newSearchResult(int shardIndex, long requestId, S
     public void testDfsWith2Shards() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, new SearchContextId(UUIDs.randomBase64UUID(), 1),
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, new SearchContextId(UUIDs.randomBase64UUID(), 2),
+            new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -65,24 +69,24 @@ public void testDfsWith2Shards() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.id() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                if (request.contextId().getId() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
+                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.id() == 2) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node2", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                } else if (request.contextId().getId() == 2) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
+                        new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F),
                             new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
                 } else {
-                    fail("no such request ID: " + request.id());
+                    fail("no such request ID: " + request.contextId());
                 }
             }
         };
@@ -114,8 +118,10 @@ public void run() throws IOException {
     public void testDfsWith1ShardFailed() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        final SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 1);
+        final SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 2);
+        results.set(0, newSearchResult(0, ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -125,18 +131,18 @@ public void testDfsWith1ShardFailed() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.id() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                if (request.contextId().getId() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
+                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(
                             new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                             new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.id() == 2) {
+                } else if (request.contextId().getId() == 2) {
                     listener.onFailure(new MockDirectoryWrapper.FakeIOException());
                 } else {
-                    fail("no such request ID: " + request.id());
+                    fail("no such request ID: " + request.contextId());
                 }
             }
         };
@@ -163,7 +169,7 @@ public void run() throws IOException {
         assertEquals(1, mockSearchPhaseContext.failures.size());
         assertTrue(mockSearchPhaseContext.failures.get(0).getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(2L));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx2));
         assertNull(responseRef.get().get(1));
     }
 
@@ -171,8 +177,10 @@ public void run() throws IOException {
     public void testFailPhaseOnException() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, new SearchContextId(UUIDs.randomBase64UUID(), 1),
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, new SearchContextId(UUIDs.randomBase64UUID(), 2),
+            new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -182,18 +190,18 @@ public void testFailPhaseOnException() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.id() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                if (request.contextId().getId() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 123),
+                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.id() == 2) {
+                } else if (request.contextId().getId() == 2) {
                    throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
                 } else {
-                    fail("no such request ID: " + request.id());
+                    fail("no such request ID: " + request.contextId());
                 }
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 5eec29dbf8039..8978f43842cfe 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -23,6 +23,8 @@
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.shard.ShardId;
@@ -35,6 +37,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
@@ -70,7 +73,7 @@ public void testShortcutQueryAndFetchOptimization() {
             numHits = 0;
         }
 
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
             @Override
             public void run() {
@@ -95,15 +98,18 @@ public void testFetchTwoDocument() {
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+        final SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
+        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        final SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 312);
+        queryResult = new QuerySearchResult(ctx2,
+            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -115,18 +121,18 @@ public void testFetchTwoDocument() {
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
                 FetchSearchResult fetchResult = new FetchSearchResult();
-                if (request.id() == 321) {
+                if (request.contextId().equals(ctx2)) {
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
-                    assertEquals(123, request.id());
+                    assertEquals(ctx1, request.contextId());
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F));
                 }
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -153,15 +159,18 @@ public void testFailFetchOneDoc() {
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+        SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
+        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 321);
+        queryResult = new QuerySearchResult(ctx2,
+            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -172,7 +181,7 @@ public void testFailFetchOneDoc() {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
-                if (request.id() == 321) {
+                if (request.contextId().getId() == 321) {
                     FetchSearchResult fetchResult = new FetchSearchResult();
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
@@ -183,7 +192,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
 
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -202,7 +211,7 @@ public void run() {
         assertEquals(1, searchResponse.getShardFailures().length);
         assertTrue(searchResponse.getShardFailures()[0].getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx1));
     }
 
     public void testFetchDocsConcurrently() throws InterruptedException {
@@ -215,8 +224,8 @@ public void testFetchDocsConcurrently() throws InterruptedException {
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP,
             mockSearchPhaseContext.getRequest(), numHits);
         for (int i = 0; i < numHits; i++) {
-            QuerySearchResult queryResult = new QuerySearchResult(i, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                null, OriginalIndices.NONE));
+            QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", i),
+                new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
             queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(i+1, i)}), i), new DocValueFormat[0]);
             queryResult.size(resultSetSize); // the size of the result set
@@ -229,14 +238,14 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                                          SearchActionListener<FetchSearchResult> listener) {
                 new Thread(() -> {
                     FetchSearchResult fetchResult = new FetchSearchResult();
-                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit((int) (request.id()+1))},
+                    fetchResult.hits(new SearchHits(new SearchHit[]{new SearchHit((int) (request.contextId().getId() + 1))},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 100F));
                     listener.onResponse(fetchResult);
                 }).start();
             }
         };
         CountDownLatch latch = new CountDownLatch(1);
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -272,15 +281,16 @@ public void testExceptionFailsPhase() {
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+        QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(new SearchContextId("", 321),
+            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -295,18 +305,18 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 if (numFetches.incrementAndGet() == 1) {
                     throw new RuntimeException("BOOM");
                 }
-                if (request.id() == 321) {
+                if (request.contextId().getId() == 321) {
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
-                    assertEquals(request, 123);
+                    assertEquals(request.contextId().getId(), 123);
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F));
                 }
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -328,15 +338,18 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = 1;
-        QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+        SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
+        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 321);
+        queryResult = new QuerySearchResult(ctx2,
+            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -348,7 +361,7 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
                 FetchSearchResult fetchResult = new FetchSearchResult();
-                if (request.id() == 321) {
+                if (request.contextId().equals(ctx2)) {
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
@@ -357,7 +370,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -375,6 +388,6 @@ public void run() {
         assertEquals(0, searchResponse.getFailedShards());
         assertEquals(2, searchResponse.getSuccessfulShards());
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx1));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index 28df802fdb409..cd060c971a4f4 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.transport.Transport;
 import org.junit.Assert;
@@ -46,7 +47,7 @@ public final class MockSearchPhaseContext implements SearchPhaseContext {
     final AtomicInteger numSuccess;
     final List<ShardSearchFailure> failures = Collections.synchronizedList(new ArrayList<>());
     SearchTransportService searchTransport;
-    final Set<Long> releasedSearchContexts = new HashSet<>();
+    final Set<SearchContextId> releasedSearchContexts = new HashSet<>();
     final SearchRequest searchRequest = new SearchRequest();
     final AtomicReference<SearchResponse> searchResponse = new AtomicReference<>();
 
@@ -135,7 +136,7 @@ public void onFailure(Exception e) {
     }
 
     @Override
-    public void sendReleaseSearchContext(long contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+    public void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
         releasedSearchContexts.add(contextId);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index bdf2026f43b40..249426057c4ac 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -21,18 +21,21 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
@@ -106,7 +109,7 @@ public void testSkipSearchShards() throws InterruptedException {
                 responseListener,
                 shardsIter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                0,
+                ClusterState.EMPTY_STATE,
                 null,
                 new ArraySearchPhaseResults<>(shardsIter.size()),
                 request.getMaxConcurrentShardRequests(),
@@ -122,7 +125,8 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
 
                     new Thread(() -> {
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
+                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()),
                         connection.getNode());
                         listener.onResponse(testSearchPhaseResult);
 
@@ -211,7 +215,7 @@ public void testLimitConcurrentShardRequests() throws InterruptedException {
                 responseListener,
                 shardsIter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                0,
+                ClusterState.EMPTY_STATE,
                 null,
                 new ArraySearchPhaseResults<>(shardsIter.size()),
                 request.getMaxConcurrentShardRequests(),
@@ -232,8 +236,8 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                             throw new AssertionError(e);
                         }
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
-                            connection.getNode());
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
+                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
                         if (shardFailures[shard.shardId().id()]) {
                             listener.onFailure(new RuntimeException());
                         } else {
@@ -277,7 +281,7 @@ public void testFanOutAndCollect() throws InterruptedException {
         DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
 
-        Map<DiscoveryNode, Set<Long>> nodeToContextMap = newConcurrentMap();
+        Map<DiscoveryNode, Set<SearchContextId>> nodeToContextMap = newConcurrentMap();
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         int numShards = randomIntBetween(1, 10);
         GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
@@ -286,7 +290,7 @@ public void testFanOutAndCollect() throws InterruptedException {
         AtomicInteger numFreedContext = new AtomicInteger();
         SearchTransportService transportService = new SearchTransportService(null, null) {
             @Override
-            public void sendFreeContext(Transport.Connection connection, long contextId, OriginalIndices originalIndices) {
+            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId, OriginalIndices originalIndices) {
                 numFreedContext.incrementAndGet();
                 assertTrue(nodeToContextMap.containsKey(connection.getNode()));
                 assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId));
@@ -314,7 +318,7 @@ public void sendFreeContext(Transport.Connection connection, long contextId, Ori
                         responseListener,
                         shardsIter,
                         new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                        0,
+                        ClusterState.EMPTY_STATE,
                         null,
                         new ArraySearchPhaseResults<>(shardsIter.size()),
                         request.getMaxConcurrentShardRequests(),
@@ -326,10 +330,10 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                 listener) {
                 assertTrue("shard: " + shard.shardId() + " has been queried twice", response.queried.add(shard.shardId()));
                 Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
-                    connection.getNode());
-                Set<Long> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
-                ids.add(testSearchPhaseResult.getRequestId());
+                TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
+                    new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
+                Set<SearchContextId> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
+                ids.add(testSearchPhaseResult.getContextId());
                 if (randomBoolean()) {
                     listener.onResponse(testSearchPhaseResult);
                 } else {
@@ -345,7 +349,7 @@ public void run() {
                         for (int i = 0; i < results.getNumShards(); i++) {
                             TestSearchPhaseResult result = results.getAtomicArray().get(i);
                             assertEquals(result.node.getId(), result.getSearchShardTarget().getNodeId());
-                            sendReleaseSearchContext(result.getRequestId(), new MockConnection(result.node), OriginalIndices.NONE);
+                            sendReleaseSearchContext(result.getContextId(), new MockConnection(result.node), OriginalIndices.NONE);
                         }
                         responseListener.onResponse(response);
                     }
@@ -419,7 +423,7 @@ public void testAllowPartialResults() throws InterruptedException {
                 responseListener,
                 shardsIter,
                 new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
-                0,
+                ClusterState.EMPTY_STATE,
                 null,
                 new ArraySearchPhaseResults<>(shardsIter.size()),
                 request.getMaxConcurrentShardRequests(),
@@ -434,8 +438,8 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                     });
                     new Thread(() -> {
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
-                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
-                            connection.getNode());
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
+                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
                         if (shardIt.remaining() > 0) {
                             numFailReplicas.incrementAndGet();
                             listener.onFailure(new RuntimeException());
@@ -513,9 +517,8 @@ public static class TestSearchResponse extends SearchResponse {
 
     public static class TestSearchPhaseResult extends SearchPhaseResult {
         final DiscoveryNode node;
-
-        TestSearchPhaseResult(long id, DiscoveryNode node) {
-            this.requestId = id;
+        TestSearchPhaseResult(SearchContextId contextId, DiscoveryNode node) {
+            this.contextId = contextId;
             this.node = node;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index f49d3a69caca0..5e65487fe5927 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -31,6 +31,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.text.Text;
@@ -50,6 +51,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.Suggest;
@@ -230,7 +232,7 @@ private static AtomicArray<SearchPhaseResult> generateQueryResults(int nShards,
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex),
                 clusterAlias, OriginalIndices.NONE);
-            QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, searchShardTarget);
+            QuerySearchResult querySearchResult = new QuerySearchResult(new SearchContextId("", shardIndex), searchShardTarget);
             final TopDocs topDocs;
             float maxScore = 0;
             if (searchHitsSize == 0) {
@@ -302,7 +304,7 @@ private static AtomicArray<SearchPhaseResult> generateFetchResults(int nShards,
             float maxScore = -1F;
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget shardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), clusterAlias, OriginalIndices.NONE);
-            FetchSearchResult fetchSearchResult = new FetchSearchResult(shardIndex, shardTarget);
+            FetchSearchResult fetchSearchResult = new FetchSearchResult(new SearchContextId("", shardIndex), shardTarget);
             List<SearchHit> searchHits = new ArrayList<>();
             for (ScoreDoc scoreDoc : mergedSearchDocs) {
                 if (scoreDoc.shardIndex == shardIndex) {
@@ -365,8 +367,8 @@ private void consumerTestCase(int numEmptyResponses) {
             numEmptyResponses --;
         }
 
-        QuerySearchResult result = new QuerySearchResult(0, new SearchShardTarget("node", new ShardId("a", "b", 0),
-            null, OriginalIndices.NONE));
+        QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 0),
+            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 1.0D, DocValueFormat.RAW,
@@ -375,7 +377,8 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(0);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
+        result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 1),
+            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 3.0D, DocValueFormat.RAW,
@@ -384,7 +387,8 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(2);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
+        result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), 1),
+            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 2.0D, DocValueFormat.RAW,
@@ -452,8 +456,8 @@ public void testConsumerConcurrently() throws InterruptedException {
             threads[i] = new Thread(() -> {
                 int number = randomIntBetween(1, 1000);
                 max.updateAndGet(prev -> Math.max(prev, number));
-                QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
-                    null, OriginalIndices.NONE));
+                QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), id),
+                    new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE));
                 result.topDocs(new TopDocsAndMaxScore(
                     new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, number)}), number),
                     new DocValueFormat[0]);
@@ -496,8 +500,8 @@ public void testConsumerOnlyAggs() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number),
                     new DocValueFormat[0]);
             InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", (double) number,
@@ -534,8 +538,8 @@ public void testConsumerOnlyHits() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(0, number)}), number), new DocValueFormat[0]);
             result.setShardIndex(i);
@@ -601,8 +605,8 @@ public void testReduceTopNWithFromOffset() {
             searchPhaseController.newSearchPhaseResults(NOOP, request, 4);
         int score = 100;
         for (int i = 0; i < 4; i++) {
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
             ScoreDoc[] docs = new ScoreDoc[3];
             for (int j = 0; j < docs.length; j++) {
                 docs[j] = new ScoreDoc(0, score--);
@@ -643,8 +647,8 @@ public void testConsumerSortByField() {
             max.updateAndGet(prev -> Math.max(prev, number));
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, new Object[]{number})};
             TopDocs topDocs = new TopFieldDocs(new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields);
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -681,8 +685,8 @@ public void testConsumerFieldCollapsing() {
             Object[] values = {randomFrom(collapseValues)};
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, values)};
             TopDocs topDocs = new CollapseTopFieldDocs("field", new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields, values);
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -714,8 +718,8 @@ public void testConsumerSuggestions() {
         int maxScorePhrase = -1;
         int maxScoreCompletion = -1;
         for (int i = 0; i < expectedNumResults; i++) {
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE));
             List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions =
                 new ArrayList<>();
             {
@@ -814,18 +818,18 @@ public void onQueryResult(int shardIndex) {
                 }
 
                 @Override
-                public void onQueryFailure(int shardIndex, Exception exc) {
+                public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
                     assertThat(shardIndex, lessThan(expectedNumResults));
                     numQueryFailureListener.incrementAndGet();
                 }
 
                 @Override
-                public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {
-                    assertEquals(numReduceListener.incrementAndGet(), version);
+                public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+                    assertEquals(numReduceListener.incrementAndGet(), reducePhase);
                 }
 
                 @Override
-                public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {
+                public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
                     totalHitsListener.set(totalHits);
                     finalAggsListener.set(aggs);
                     numReduceListener.incrementAndGet();
@@ -840,8 +844,8 @@ public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggr
                 threads[i] = new Thread(() -> {
                     int number = randomIntBetween(1, 1000);
                     max.updateAndGet(prev -> Math.max(prev, number));
-                    QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
-                        null, OriginalIndices.NONE));
+                    QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.randomBase64UUID(), id),
+                        new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE));
                     result.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[]{new ScoreDoc(0, number)}), number),
                         new DocValueFormat[0]);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
index 1a6bb6263fb6b..3f9269afd9e78 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
@@ -24,11 +24,13 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.util.ArrayList;
@@ -37,6 +39,7 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -139,7 +142,8 @@ private static void testCase(NodeClient client, SearchRequest request,
         CountDownLatch latch = new CountDownLatch(1);
         SearchProgressActionListener listener = new SearchProgressActionListener() {
             @Override
-            public void onListShards(List<SearchShard> shards, boolean fetchPhase) {
+            public void onListShards(List<SearchShard> shards, List<SearchShard> skippedShards,
+                                     SearchResponse.Clusters clusters, boolean fetchPhase) {
                 shardsListener.set(shards);
                 assertEquals(fetchPhase, hasFetchPhase);
             }
@@ -151,7 +155,7 @@ public void onQueryResult(int shardIndex) {
             }
 
             @Override
-            public void onQueryFailure(int shardIndex, Exception exc) {
+            public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
                 assertThat(shardIndex, lessThan(shardsListener.get().size()));
                 numQueryFailures.incrementAndGet();
             }
@@ -169,12 +173,12 @@ public void onFetchFailure(int shardIndex, Exception exc) {
             }
 
             @Override
-            public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {
+            public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
                 numReduces.incrementAndGet();
             }
 
             @Override
-            public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {
+            public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
                 numReduces.incrementAndGet();
             }
 
@@ -189,7 +193,14 @@ public void onFailure(Exception e) {
                 throw new AssertionError();
             }
         };
-        client.executeSearchLocally(request, listener);
+        client.executeLocally(SearchAction.INSTANCE, new SearchRequest(request) {
+            @Override
+            public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+                SearchTask task = super.createTask(id, type, action, parentTaskId, headers);
+                task.setProgressListener(listener);
+                return task;
+            }
+        }, listener);
         latch.await();
 
         assertThat(shardsListener.get(), equalTo(expectedShards));
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
index 2494b45f61174..49eb7673592bf 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
@@ -23,12 +23,14 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 
@@ -48,11 +50,11 @@ public class SearchScrollAsyncActionTests extends ESTestCase {
     public void testSendRequestsToNodes() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", 1),
-            new ScrollIdForNode(null, "node2", 2),
-            new ScrollIdForNode(null, "node3", 17),
-            new ScrollIdForNode(null, "node1", 0),
-            new ScrollIdForNode(null, "node3", 0));
+            new ScrollIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 1)),
+            new ScrollIdForNode(null, "node2", new SearchContextId(UUIDs.randomBase64UUID(), 2)),
+            new ScrollIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 17)),
+            new ScrollIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 0)),
+            new ScrollIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -73,7 +75,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                 {
                     new Thread(() -> {
                         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
                         testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                             new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                         searchActionListener.onResponse(testSearchPhaseResult);
@@ -109,7 +111,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         ScrollIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
-            assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+            assertEquals(context[i].getContextId(), results.get(i).getContextId());
             assertEquals(context[i].getNode(), results.get(i).node.getId());
         }
     }
@@ -117,11 +119,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
     public void testFailNextPhase() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", 1),
-            new ScrollIdForNode(null, "node2", 2),
-            new ScrollIdForNode(null, "node3", 17),
-            new ScrollIdForNode(null, "node1", 0),
-            new ScrollIdForNode(null, "node3", 0));
+            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ScrollIdForNode(null, "node2", new SearchContextId("a", 2)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("b", 17)),
+            new ScrollIdForNode(null, "node1", new SearchContextId("c", 0)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("d", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -164,7 +166,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                 {
                     new Thread(() -> {
                         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
                         testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                             new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                         searchActionListener.onResponse(testSearchPhaseResult);
@@ -200,18 +202,18 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         ScrollIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
-            assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+            assertEquals(context[i].getContextId(), results.get(i).getContextId());
             assertEquals(context[i].getNode(), results.get(i).node.getId());
         }
     }
 
     public void testNodeNotAvailable() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", 1),
-            new ScrollIdForNode(null, "node2", 2),
-            new ScrollIdForNode(null, "node3", 17),
-            new ScrollIdForNode(null, "node1", 0),
-            new ScrollIdForNode(null, "node3", 0));
+            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("", 17)),
+            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
         // node2 is not available
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -237,7 +239,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                     }
                     new Thread(() -> {
                         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                            new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
                         testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                             new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                         searchActionListener.onResponse(testSearchPhaseResult);
@@ -279,7 +281,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
                 assertNull(results.get(i));
             } else {
                 assertNotNull(results.get(i));
-                assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+                assertEquals(context[i].getContextId(), results.get(i).getContextId());
                 assertEquals(context[i].getNode(), results.get(i).node.getId());
             }
         }
@@ -287,11 +289,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testShardFailures() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", 1),
-            new ScrollIdForNode(null, "node2", 2),
-            new ScrollIdForNode(null, "node3", 17),
-            new ScrollIdForNode(null, "node1", 0),
-            new ScrollIdForNode(null, "node3", 0));
+            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("",17)),
+            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -311,11 +313,11 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro
                                                    SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener)
                 {
                     new Thread(() -> {
-                        if (internalRequest.id() == 17) {
+                        if (internalRequest.contextId().getId() == 17) {
                             searchActionListener.onFailure(new IllegalArgumentException("BOOM on shard"));
                         } else {
                             SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult =
-                                new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode());
+                                new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
                             testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(),
                                 new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                             searchActionListener.onResponse(testSearchPhaseResult);
@@ -353,11 +355,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
         ScrollIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
-            if (context[i].getScrollId() == 17) {
+            if (context[i].getContextId().getId() == 17) {
                 assertNull(results.get(i));
             } else {
                 assertNotNull(results.get(i));
-                assertEquals(context[i].getScrollId(), results.get(i).getRequestId());
+                assertEquals(context[i].getContextId(), results.get(i).getContextId());
                 assertEquals(context[i].getNode(), results.get(i).node.getId());
             }
         }
@@ -365,11 +367,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testAllShardsFailed() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", 1),
-            new ScrollIdForNode(null, "node2", 2),
-            new ScrollIdForNode(null, "node3", 17),
-            new ScrollIdForNode(null, "node1", 0),
-            new ScrollIdForNode(null, "node3", 0));
+            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("", 17)),
+            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
+            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
index 466f56a792491..f661cf8e7e8a7 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -32,6 +33,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -57,12 +59,13 @@ public void testSerialization() throws Exception {
 
     public void testInternalScrollSearchRequestSerialization() throws IOException {
         SearchScrollRequest searchScrollRequest = createSearchScrollRequest();
-        InternalScrollSearchRequest internalScrollSearchRequest = new InternalScrollSearchRequest(searchScrollRequest, randomLong());
+        InternalScrollSearchRequest internalScrollSearchRequest =
+            new InternalScrollSearchRequest(searchScrollRequest, new SearchContextId(UUIDs.randomBase64UUID(), randomLong()));
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             internalScrollSearchRequest.writeTo(output);
             try (StreamInput in = output.bytes().streamInput()) {
                 InternalScrollSearchRequest deserializedRequest = new InternalScrollSearchRequest(in);
-                assertEquals(deserializedRequest.id(), internalScrollSearchRequest.id());
+                assertEquals(deserializedRequest.contextId().getId(), internalScrollSearchRequest.contextId().getId());
                 assertEquals(deserializedRequest.scroll(), internalScrollSearchRequest.scroll());
                 assertNotSame(deserializedRequest, internalScrollSearchRequest);
             }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java
new file mode 100644
index 0000000000000..8ee1718023e69
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchShardTests.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class SearchShardTests extends ESTestCase {
+
+   public void testEqualsAndHashcode() {
+       String index = randomAlphaOfLengthBetween(5, 10);
+       SearchShard searchShard = new SearchShard(randomBoolean() ? null : randomAlphaOfLengthBetween(3, 10),
+           new ShardId(index, index + "-uuid", randomIntBetween(0, 1024)));
+       EqualsHashCodeTestUtils.checkEqualsAndHashCode(searchShard,
+           s -> new SearchShard(s.getClusterAlias(), s.getShardId()),
+           s -> {
+            if (randomBoolean()) {
+                return new SearchShard(s.getClusterAlias() == null ? randomAlphaOfLengthBetween(3, 10) : null, s.getShardId());
+            } else {
+                String indexName = s.getShardId().getIndexName();
+                int shardId = s.getShardId().getId();
+                if (randomBoolean()) {
+                    indexName += randomAlphaOfLength(5);
+                } else {
+                    shardId += randomIntBetween(1, 1024);
+                }
+                return new SearchShard(s.getClusterAlias(), new ShardId(indexName, indexName + "-uuid", shardId));
+            }
+           });
+   }
+
+   public void testCompareTo() {
+       List<SearchShard> searchShards = new ArrayList<>();
+       Index index0 = new Index("index0", "index0-uuid");
+       Index index1 = new Index("index1", "index1-uuid");
+       searchShards.add(new SearchShard(null, new ShardId(index0, 0)));
+       searchShards.add(new SearchShard(null, new ShardId(index1, 0)));
+       searchShards.add(new SearchShard(null, new ShardId(index0, 1)));
+       searchShards.add(new SearchShard(null, new ShardId(index1, 1)));
+       searchShards.add(new SearchShard(null, new ShardId(index0, 2)));
+       searchShards.add(new SearchShard(null, new ShardId(index1, 2)));
+       searchShards.add(new SearchShard("", new ShardId(index0, 0)));
+       searchShards.add(new SearchShard("", new ShardId(index1, 0)));
+       searchShards.add(new SearchShard("", new ShardId(index0, 1)));
+       searchShards.add(new SearchShard("", new ShardId(index1, 1)));
+
+       searchShards.add(new SearchShard("remote0", new ShardId(index0, 0)));
+       searchShards.add(new SearchShard("remote0", new ShardId(index1, 0)));
+       searchShards.add(new SearchShard("remote0", new ShardId(index0, 1)));
+       searchShards.add(new SearchShard("remote0", new ShardId(index0, 2)));
+       searchShards.add(new SearchShard("remote1", new ShardId(index0, 0)));
+       searchShards.add(new SearchShard("remote1", new ShardId(index1, 0)));
+       searchShards.add(new SearchShard("remote1", new ShardId(index0, 1)));
+       searchShards.add(new SearchShard("remote1", new ShardId(index1, 1)));
+
+       List<SearchShard> sorted = new ArrayList<>(searchShards);
+       Collections.sort(sorted);
+       assertEquals(searchShards, sorted);
+   }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
index a666f45c34311..c9d58c7dc9090 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -24,10 +24,13 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class TransportSearchHelperTests extends ESTestCase {
 
     public void testParseScrollId() throws IOException {
@@ -35,29 +38,48 @@ public void testParseScrollId() throws IOException {
         DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("x", 1), node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("y", 12), node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null));
-        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3);
+        SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("z", 42), node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
         array.setOnce(2, testSearchPhaseResult3);
 
-        String scrollId = TransportSearchHelper.buildScrollId(array);
+        boolean includeUUID = randomBoolean();
+        String scrollId = TransportSearchHelper.buildScrollId(array, includeUUID);
         ParsedScrollId parseScrollId = TransportSearchHelper.parseScrollId(scrollId);
         assertEquals(3, parseScrollId.getContext().length);
         assertEquals("node_1", parseScrollId.getContext()[0].getNode());
         assertEquals("cluster_x", parseScrollId.getContext()[0].getClusterAlias());
-        assertEquals(1, parseScrollId.getContext()[0].getScrollId());
+        assertEquals(1, parseScrollId.getContext()[0].getContextId().getId());
+        if (includeUUID) {
+            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo("x"));
+        } else {
+            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo(""));
+        }
 
         assertEquals("node_2", parseScrollId.getContext()[1].getNode());
         assertEquals("cluster_y", parseScrollId.getContext()[1].getClusterAlias());
-        assertEquals(12, parseScrollId.getContext()[1].getScrollId());
+        assertEquals(12, parseScrollId.getContext()[1].getContextId().getId());
+        if (includeUUID) {
+            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo("y"));
+        } else {
+            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo(""));
+        }
 
         assertEquals("node_3", parseScrollId.getContext()[2].getNode());
         assertNull(parseScrollId.getContext()[2].getClusterAlias());
-        assertEquals(42, parseScrollId.getContext()[2].getScrollId());
+        assertEquals(42, parseScrollId.getContext()[2].getContextId().getId());
+        if (includeUUID) {
+            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo("z"));
+        } else {
+            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo(""));
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index eb2f1c9d845a1..afb63edfd15e1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.cluster;
 
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -31,10 +33,13 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
@@ -72,7 +77,6 @@
 public class NodeConnectionsServiceTests extends ESTestCase {
 
     private ThreadPool threadPool;
-    private MockTransport transport;
     private TransportService transportService;
     private Map<DiscoveryNode, CheckedRunnable<Exception>> nodeConnectionBlocks;
 
@@ -301,6 +305,116 @@ public void testOnlyBlocksOnConnectionsToNewNodes() throws Exception {
         }
     }
 
+    @TestLogging(reason="testing that DEBUG-level logging is reasonable", value="org.elasticsearch.cluster.NodeConnectionsService:DEBUG")
+    public void testDebugLogging() throws IllegalAccessException {
+        final DeterministicTaskQueue deterministicTaskQueue
+            = new DeterministicTaskQueue(builder().put(NODE_NAME_SETTING.getKey(), "node").build(), random());
+
+        MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool());
+        TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool());
+        transportService.start();
+        transportService.acceptIncomingRequests();
+
+        final NodeConnectionsService service
+            = new NodeConnectionsService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), transportService);
+        service.start();
+
+        final List<DiscoveryNode> allNodes = generateNodes();
+        final DiscoveryNodes targetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
+        service.connectToNodes(targetNodes, () -> {});
+        deterministicTaskQueue.runAllRunnableTasks();
+
+        // periodic reconnections to unexpectedly-disconnected nodes are logged
+        final Set<DiscoveryNode> disconnectedNodes = new HashSet<>(randomSubsetOf(allNodes));
+        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
+            transportService.disconnectFromNode(disconnectedNode);
+        }
+        MockLogAppender appender = new MockLogAppender();
+        try {
+            appender.start();
+            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            for (DiscoveryNode targetNode : targetNodes) {
+                if (disconnectedNodes.contains(targetNode)) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                } else {
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                }
+            }
+
+            runTasksUntil(deterministicTaskQueue, CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(Settings.EMPTY).millis());
+            appender.assertAllExpectationsMatched();
+        } finally {
+            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            appender.stop();
+        }        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
+            transportService.disconnectFromNode(disconnectedNode);
+        }
+
+        // changes to the expected set of nodes are logged, including reconnections to any unexpectedly-disconnected nodes
+        final DiscoveryNodes newTargetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
+        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
+            transportService.disconnectFromNode(disconnectedNode);
+        }
+        appender = new MockLogAppender();
+        try {
+            appender.start();
+            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            for (DiscoveryNode targetNode : targetNodes) {
+                if (disconnectedNodes.contains(targetNode) && newTargetNodes.get(targetNode.getId()) != null) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                } else {
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                }
+                if (newTargetNodes.get(targetNode.getId()) == null) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("disconnected from " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "disconnected from " + targetNode));
+                }
+            }
+            for (DiscoveryNode targetNode : newTargetNodes) {
+                appender.addExpectation(new MockLogAppender.UnseenEventExpectation("disconnected from " + targetNode,
+                    "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                    "disconnected from " + targetNode));
+                if (targetNodes.get(targetNode.getId()) == null) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                }
+            }
+
+            service.disconnectFromNodesExcept(newTargetNodes);
+            service.connectToNodes(newTargetNodes, () -> {});
+            deterministicTaskQueue.runAllRunnableTasks();
+            appender.assertAllExpectationsMatched();
+        } finally {
+            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            appender.stop();
+        }
+    }
+
     private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long endTimeMillis) {
         while (deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) {
             if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) {
@@ -339,9 +453,8 @@ public void setUp() throws Exception {
         super.setUp();
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
         this.threadPool = threadPool;
-        this.transport = new MockTransport(threadPool);
         nodeConnectionBlocks = newConcurrentMap();
-        transportService = new TestTransportService(transport, threadPool);
+        transportService = new TestTransportService(new MockTransport(threadPool), threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
index ef7567ea5df91..a800a7adf093b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
@@ -68,7 +68,7 @@ public void testJoinDeduplication() {
         // check that sending a join to node1 works
         Optional<Join> optionalJoin1 = randomBoolean() ? Optional.empty() :
             Optional.of(new Join(localNode, node1, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
-        joinHelper.sendJoinRequest(node1, optionalJoin1);
+        joinHelper.sendJoinRequest(node1, 0L, optionalJoin1);
         CapturedRequest[] capturedRequests1 = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests1.length, equalTo(1));
         CapturedRequest capturedRequest1 = capturedRequests1[0];
@@ -79,14 +79,14 @@ public void testJoinDeduplication() {
         // check that sending a join to node2 works
         Optional<Join> optionalJoin2 = randomBoolean() ? Optional.empty() :
             Optional.of(new Join(localNode, node2, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
-        joinHelper.sendJoinRequest(node2, optionalJoin2);
+        joinHelper.sendJoinRequest(node2, 0L, optionalJoin2);
         CapturedRequest[] capturedRequests2 = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests2.length, equalTo(1));
         CapturedRequest capturedRequest2 = capturedRequests2[0];
         assertEquals(node2, capturedRequest2.node);
 
         // check that sending another join to node1 is a noop as the previous join is still in progress
-        joinHelper.sendJoinRequest(node1, optionalJoin1);
+        joinHelper.sendJoinRequest(node1, 0L, optionalJoin1);
         assertThat(capturingTransport.getCapturedRequestsAndClear().length, equalTo(0));
 
         // complete the previous join to node1
@@ -97,7 +97,7 @@ public void testJoinDeduplication() {
         }
 
         // check that sending another join to node1 now works again
-        joinHelper.sendJoinRequest(node1, optionalJoin1);
+        joinHelper.sendJoinRequest(node1, 0L, optionalJoin1);
         CapturedRequest[] capturedRequests1a = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests1a.length, equalTo(1));
         CapturedRequest capturedRequest1a = capturedRequests1a[0];
@@ -106,7 +106,7 @@ public void testJoinDeduplication() {
         // check that sending another join to node2 works if the optionalJoin is different
         Optional<Join> optionalJoin2a = optionalJoin2.isPresent() && randomBoolean() ? Optional.empty() :
             Optional.of(new Join(localNode, node2, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
-        joinHelper.sendJoinRequest(node2, optionalJoin2a);
+        joinHelper.sendJoinRequest(node2, 0L, optionalJoin2a);
         CapturedRequest[] capturedRequests2a = capturingTransport.getCapturedRequestsAndClear();
         assertThat(capturedRequests2a.length, equalTo(1));
         CapturedRequest capturedRequest2a = capturedRequests2a[0];
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java
index cb89c7c0fea33..fb99649892ae2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java
@@ -179,13 +179,18 @@ public void testJoinRequestEqualsHashCodeSerialization() {
         Join initialJoin = new Join(createNode(randomAlphaOfLength(10)), createNode(randomAlphaOfLength(10)), randomNonNegativeLong(),
             randomNonNegativeLong(), randomNonNegativeLong());
         JoinRequest initialJoinRequest = new JoinRequest(initialJoin.getSourceNode(),
-            randomBoolean() ? Optional.empty() : Optional.of(initialJoin));
+            randomNonNegativeLong(), randomBoolean() ? Optional.empty() : Optional.of(initialJoin));
         // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(initialJoinRequest,
                 (CopyFunction<JoinRequest>) joinRequest -> copyWriteable(joinRequest, writableRegistry(), JoinRequest::new),
             joinRequest -> {
                 if (randomBoolean() && joinRequest.getOptionalJoin().isPresent() == false) {
-                    return new JoinRequest(createNode(randomAlphaOfLength(20)), joinRequest.getOptionalJoin());
+                    return new JoinRequest(createNode(randomAlphaOfLength(10)),
+                        joinRequest.getMinimumTerm(), joinRequest.getOptionalJoin());
+                } else if (randomBoolean()) {
+                    return new JoinRequest(joinRequest.getSourceNode(),
+                        randomValueOtherThan(joinRequest.getMinimumTerm(), ESTestCase::randomNonNegativeLong),
+                        joinRequest.getOptionalJoin());
                 } else {
                     // change OptionalJoin
                     final Optional<Join> newOptionalJoin;
@@ -195,7 +200,7 @@ public void testJoinRequestEqualsHashCodeSerialization() {
                         newOptionalJoin = Optional.of(new Join(joinRequest.getSourceNode(), createNode(randomAlphaOfLength(10)),
                             randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
                     }
-                    return new JoinRequest(joinRequest.getSourceNode(), newOptionalJoin);
+                    return new JoinRequest(joinRequest.getSourceNode(), joinRequest.getMinimumTerm(), newOptionalJoin);
                 }
             });
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index 60575f7487676..ac3075597dc57 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -74,6 +74,7 @@
 import static org.elasticsearch.transport.TransportService.HANDSHAKE_ACTION_NAME;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class NodeJoinTests extends ESTestCase {
 
@@ -278,7 +279,8 @@ public void testJoinWithHigherTermElectsLeader() {
         assertFalse(isLocalNodeElectedMaster());
         assertNull(coordinator.getStateForMasterService().nodes().getMasterNodeId());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture fut = joinNodeAsync(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
+        SimpleFuture fut = joinNodeAsync(new JoinRequest(node1, newTerm,
+            Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
         assertEquals(Coordinator.Mode.LEADER, coordinator.getMode());
         assertNull(coordinator.getStateForMasterService().nodes().getMasterNodeId());
         deterministicTaskQueue.runAllRunnableTasks();
@@ -298,7 +300,8 @@ public void testJoinWithHigherTermButBetterStateGetsRejected() {
         long newTerm = initialTerm + randomLongBetween(1, 10);
         long higherVersion = initialVersion + randomLongBetween(1, 10);
         expectThrows(CoordinationStateRejectedException.class,
-            () -> joinNodeAndRun(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion)))));
+            () -> joinNodeAndRun(new JoinRequest(node1, newTerm,
+                Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion)))));
         assertFalse(isLocalNodeElectedMaster());
     }
 
@@ -312,7 +315,7 @@ public void testJoinWithHigherTermButBetterStateStillElectsMasterThroughSelfJoin
         assertFalse(isLocalNodeElectedMaster());
         long newTerm = initialTerm + randomLongBetween(1, 10);
         long higherVersion = initialVersion + randomLongBetween(1, 10);
-        joinNodeAndRun(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion))));
+        joinNodeAndRun(new JoinRequest(node1, newTerm, Optional.of(new Join(node1, node0, newTerm, initialTerm, higherVersion))));
         assertTrue(isLocalNodeElectedMaster());
     }
 
@@ -325,14 +328,32 @@ public void testJoinElectedLeader() {
             new VotingConfiguration(Collections.singleton(node0.getId()))));
         assertFalse(isLocalNodeElectedMaster());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        joinNodeAndRun(new JoinRequest(node0, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
+        joinNodeAndRun(new JoinRequest(node0, newTerm, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
         assertFalse(clusterStateHasNode(node1));
-        joinNodeAndRun(new JoinRequest(node1, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
+        joinNodeAndRun(new JoinRequest(node1, newTerm, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
         assertTrue(clusterStateHasNode(node1));
     }
 
+    public void testJoinElectedLeaderWithHigherTerm() {
+        DiscoveryNode node0 = newNode(0, true);
+        DiscoveryNode node1 = newNode(1, true);
+        long initialTerm = randomLongBetween(1, 10);
+        long initialVersion = randomLongBetween(1, 10);
+        setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
+            new VotingConfiguration(Collections.singleton(node0.getId()))));
+        long newTerm = initialTerm + randomLongBetween(1, 10);
+
+        joinNodeAndRun(new JoinRequest(node0, newTerm, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
+        assertTrue(isLocalNodeElectedMaster());
+
+        long newerTerm = newTerm + randomLongBetween(1, 10);
+        joinNodeAndRun(new JoinRequest(node1, newerTerm, Optional.empty()));
+        assertThat(coordinator.getCurrentTerm(), greaterThanOrEqualTo(newerTerm));
+        assertTrue(isLocalNodeElectedMaster());
+    }
+
     public void testJoinAccumulation() {
         DiscoveryNode node0 = newNode(0, true);
         DiscoveryNode node1 = newNode(1, true);
@@ -343,17 +364,17 @@ public void testJoinAccumulation() {
             new VotingConfiguration(Collections.singleton(node2.getId()))));
         assertFalse(isLocalNodeElectedMaster());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture futNode0 = joinNodeAsync(new JoinRequest(node0, Optional.of(
+        SimpleFuture futNode0 = joinNodeAsync(new JoinRequest(node0, newTerm, Optional.of(
             new Join(node0, node0, newTerm, initialTerm, initialVersion))));
         deterministicTaskQueue.runAllRunnableTasks();
         assertFalse(futNode0.isDone());
         assertFalse(isLocalNodeElectedMaster());
-        SimpleFuture futNode1 = joinNodeAsync(new JoinRequest(node1, Optional.of(
+        SimpleFuture futNode1 = joinNodeAsync(new JoinRequest(node1, newTerm, Optional.of(
             new Join(node1, node0, newTerm, initialTerm, initialVersion))));
         deterministicTaskQueue.runAllRunnableTasks();
         assertFalse(futNode1.isDone());
         assertFalse(isLocalNodeElectedMaster());
-        joinNodeAndRun(new JoinRequest(node2, Optional.of(new Join(node2, node0, newTerm, initialTerm, initialVersion))));
+        joinNodeAndRun(new JoinRequest(node2, newTerm, Optional.of(new Join(node2, node0, newTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
         assertTrue(clusterStateHasNode(node1));
         assertTrue(clusterStateHasNode(node2));
@@ -372,7 +393,7 @@ public void testJoinFollowerWithHigherTerm() throws Exception {
         handleStartJoinFrom(node1, newTerm);
         handleFollowerCheckFrom(node1, newTerm);
         long newerTerm = newTerm + randomLongBetween(1, 10);
-        joinNodeAndRun(new JoinRequest(node1,
+        joinNodeAndRun(new JoinRequest(node1, newerTerm,
             Optional.of(new Join(node1, node0, newerTerm, initialTerm, initialVersion))));
         assertTrue(isLocalNodeElectedMaster());
     }
@@ -447,7 +468,7 @@ public void testJoinFollowerFails() throws Exception {
         handleStartJoinFrom(node1, newTerm);
         handleFollowerCheckFrom(node1, newTerm);
         assertThat(expectThrows(CoordinationStateRejectedException.class,
-            () -> joinNodeAndRun(new JoinRequest(node1, Optional.empty()))).getMessage(),
+            () -> joinNodeAndRun(new JoinRequest(node1, newTerm, Optional.empty()))).getMessage(),
             containsString("join target is a follower"));
         assertFalse(isLocalNodeElectedMaster());
     }
@@ -460,7 +481,8 @@ public void testBecomeFollowerFailsPendingJoin() throws Exception {
         setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
             new VotingConfiguration(Collections.singleton(node1.getId()))));
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture fut = joinNodeAsync(new JoinRequest(node0, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
+        SimpleFuture fut = joinNodeAsync(new JoinRequest(node0, newTerm,
+            Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
         deterministicTaskQueue.runAllRunnableTasks();
         assertFalse(fut.isDone());
         assertFalse(isLocalNodeElectedMaster());
@@ -501,7 +523,7 @@ public void testConcurrentJoining() {
         logger.info("Successful voting nodes: {}", successfulNodes);
 
         List<JoinRequest> correctJoinRequests = successfulNodes.stream().map(
-            node -> new JoinRequest(node, Optional.of(new Join(node, localNode, newTerm, initialTerm, initialVersion))))
+            node -> new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode, newTerm, initialTerm, initialVersion))))
             .collect(Collectors.toList());
 
         List<DiscoveryNode> possiblyUnsuccessfulNodes = new ArrayList<>(allNodes);
@@ -512,15 +534,15 @@ public void testConcurrentJoining() {
         List<JoinRequest> possiblyFailingJoinRequests = possiblyUnsuccessfulNodes.stream().map(node -> {
             if (randomBoolean()) {
                 // a correct request
-                return new JoinRequest(node, Optional.of(new Join(node, localNode,
+                return new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode,
                     newTerm, initialTerm, initialVersion)));
             } else if (randomBoolean()) {
                 // term too low
-                return new JoinRequest(node, Optional.of(new Join(node, localNode,
+                return new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode,
                     randomLongBetween(0, initialTerm), initialTerm, initialVersion)));
             } else {
                 // better state
-                return new JoinRequest(node, Optional.of(new Join(node, localNode,
+                return new JoinRequest(node, newTerm, Optional.of(new Join(node, localNode,
                     newTerm, initialTerm, initialVersion + randomLongBetween(1, 10))));
             }
         }).collect(Collectors.toList());
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
index e44805744aa5c..f58753c42c4da 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
@@ -105,7 +105,7 @@ public void testHandleNodeJoin_incompatibleClusterState()
         final CompletableFuture<Throwable> future = new CompletableFuture<>();
         DiscoveryNode node = state.nodes().getLocalNode();
 
-        coordinator.sendValidateJoinRequest(stateWithCustomMetaData, new JoinRequest(node, Optional.empty()),
+        coordinator.sendValidateJoinRequest(stateWithCustomMetaData, new JoinRequest(node, 0L, Optional.empty()),
                 new JoinHelper.JoinCallback() {
             @Override
             public void onSuccess() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
new file mode 100644
index 0000000000000..e8d654603bc12
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ComponentTemplateMetadataTests extends AbstractNamedWriteableTestCase<ComponentTemplateMetadata> {
+
+    @Override
+    protected ComponentTemplateMetadata createTestInstance() {
+        int count = randomIntBetween(0, 3);
+        Map<String, ComponentTemplate> templateMap = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            templateMap.put(randomAlphaOfLength(4), ComponentTemplateTests.randomInstance());
+        }
+        return new ComponentTemplateMetadata(templateMap);
+    }
+
+    @Override
+    protected ComponentTemplateMetadata mutateInstance(ComponentTemplateMetadata instance) throws IOException {
+        if (instance.componentTemplates().size() == 0) {
+            // Not really much to mutate, so just generate a new one
+            return randomValueOtherThan(instance, this::createTestInstance);
+        }
+        Map<String, ComponentTemplate> templates = new HashMap<>(instance.componentTemplates());
+        Map.Entry<String, ComponentTemplate> newTemplate = instance.componentTemplates().entrySet().iterator().next();
+        if (randomBoolean()) {
+            // Change the key
+            templates.put(randomAlphaOfLength(4), newTemplate.getValue());
+        } else {
+            // Change the value
+            templates.put(newTemplate.getKey(), ComponentTemplateTests.mutateTemplate(newTemplate.getValue()));
+        }
+        return new ComponentTemplateMetadata(templates);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.singletonList(new NamedWriteableRegistry.Entry(ComponentTemplateMetadata.class,
+            ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new)));
+    }
+
+    @Override
+    protected Class<ComponentTemplateMetadata> categoryClass() {
+        return ComponentTemplateMetadata.class;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
new file mode 100644
index 0000000000000..f0c1b523ac9c1
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public class ComponentTemplateTests extends AbstractDiffableSerializationTestCase<ComponentTemplate> {
+    @Override
+    protected ComponentTemplate makeTestChanges(ComponentTemplate testInstance) {
+        try {
+            return mutateInstance(testInstance);
+        } catch (IOException e) {
+            logger.error(e);
+            fail("mutating should not throw an exception, but got: " + e);
+            return null;
+        }
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<ComponentTemplate>> diffReader() {
+        return ComponentTemplate::readComponentTemplateDiffFrom;
+    }
+
+    @Override
+    protected ComponentTemplate doParseInstance(XContentParser parser) throws IOException {
+        return ComponentTemplate.parse(parser);
+    }
+
+    @Override
+    protected Writeable.Reader<ComponentTemplate> instanceReader() {
+        return ComponentTemplate::new;
+    }
+
+    @Override
+    protected ComponentTemplate createTestInstance() {
+        return randomInstance();
+    }
+
+    public static ComponentTemplate randomInstance() {
+        Settings settings = null;
+        CompressedXContent mappings = null;
+        Map<String, AliasMetaData> aliases = null;
+        if (randomBoolean()) {
+            settings = randomSettings();
+        }
+        if (randomBoolean()) {
+            mappings = randomMappings();
+        }
+        if (randomBoolean()) {
+            aliases = randomAliases();
+        }
+        ComponentTemplate.Template template = new ComponentTemplate.Template(settings, mappings, aliases);
+
+        Map<String, Object> meta = null;
+        if (randomBoolean()) {
+            meta = randomMeta();
+        }
+        return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta);
+    }
+
+    private static Map<String, AliasMetaData> randomAliases() {
+        String aliasName = randomAlphaOfLength(5);
+        AliasMetaData aliasMeta = AliasMetaData.builder(aliasName)
+            .filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
+            .routing(randomBoolean() ? null : randomAlphaOfLength(3))
+            .isHidden(randomBoolean() ? null : randomBoolean())
+            .writeIndex(randomBoolean() ? null : randomBoolean())
+            .build();
+        return Collections.singletonMap(aliasName, aliasMeta);
+    }
+
+    private static CompressedXContent randomMappings() {
+        try {
+            return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}");
+        } catch (IOException e) {
+            fail("got an IO exception creating fake mappings: " + e);
+            return null;
+        }
+    }
+
+    private static Settings randomSettings() {
+        return Settings.builder()
+            .put(randomAlphaOfLength(4), randomAlphaOfLength(10))
+            .build();
+    }
+
+    private static Map<String, Object> randomMeta() {
+        if (randomBoolean()) {
+            return Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4));
+        } else {
+            return Collections.singletonMap(randomAlphaOfLength(5),
+                Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4)));
+        }
+    }
+
+    @Override
+    protected ComponentTemplate mutateInstance(ComponentTemplate orig) throws IOException {
+        return mutateTemplate(orig);
+    }
+
+    public static ComponentTemplate mutateTemplate(ComponentTemplate orig) {
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                switch (randomIntBetween(0, 2)) {
+                    case 0:
+                        ComponentTemplate.Template ot = orig.template();
+                        return new ComponentTemplate(
+                            new ComponentTemplate.Template(randomValueOtherThan(ot.settings(), ComponentTemplateTests::randomSettings),
+                                ot.mappings(), ot.aliases()),
+                            orig.version(), orig.metadata());
+                    case 1:
+                        ComponentTemplate.Template ot2 = orig.template();
+                        return new ComponentTemplate(
+                            new ComponentTemplate.Template(ot2.settings(),
+                                randomValueOtherThan(ot2.mappings(), ComponentTemplateTests::randomMappings), ot2.aliases()),
+                            orig.version(), orig.metadata());
+                    case 2:
+                        ComponentTemplate.Template ot3 = orig.template();
+                        return new ComponentTemplate(
+                            new ComponentTemplate.Template(ot3.settings(), ot3.mappings(),
+                                randomValueOtherThan(ot3.aliases(), ComponentTemplateTests::randomAliases)),
+                            orig.version(), orig.metadata());
+                    default:
+                        throw new IllegalStateException("illegal randomization branch");
+                }
+            case 1:
+                return new ComponentTemplate(orig.template(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong),
+                    orig.metadata());
+            case 2:
+                return new ComponentTemplate(orig.template(), orig.version(),
+                    randomValueOtherThan(orig.metadata(), ComponentTemplateTests::randomMeta));
+            default:
+                throw new IllegalStateException("illegal randomization branch");
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 5bd981f246a56..1d0bd3e36c57b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -723,7 +724,7 @@ public void testFindMappingsWithFilters() throws IOException {
         }
     }
 
-    private IndexMetaData.Builder buildIndexMetaData(String name, String alias, Boolean writeIndex) {
+    private static IndexMetaData.Builder buildIndexMetaData(String name, String alias, Boolean writeIndex) {
         return IndexMetaData.builder(name)
             .settings(settings(Version.CURRENT)).creationDate(randomNonNegativeLong())
             .putAlias(AliasMetaData.builder(alias).writeIndex(writeIndex))
@@ -906,4 +907,37 @@ public void testBuilderRejectsNullInCustoms() {
         final ImmutableOpenMap<String, MetaData.Custom> map = mapBuilder.build();
         assertThat(expectThrows(NullPointerException.class, () -> builder.customs(map)).getMessage(), containsString(key));
     }
+
+    public void testSerialization() throws IOException {
+        final MetaData orig = randomMetaData();
+        final BytesStreamOutput out = new BytesStreamOutput();
+        orig.writeTo(out);
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+        final MetaData fromStreamMeta = MetaData.readFrom(new NamedWriteableAwareStreamInput(out.bytes().streamInput(),
+            namedWriteableRegistry));
+        assertTrue(MetaData.isGlobalStateEquals(orig, fromStreamMeta));
+    }
+
+    public static MetaData randomMetaData() {
+        return MetaData.builder()
+            .put(buildIndexMetaData("index", "alias", randomBoolean() ? null : randomBoolean()).build(), randomBoolean())
+            .put(IndexTemplateMetaData.builder("template" + randomAlphaOfLength(3))
+                .patterns(Arrays.asList("bar-*", "foo-*"))
+                .settings(Settings.builder()
+                    .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
+                    .build())
+                .build())
+            .persistentSettings(Settings.builder()
+                .put("setting" + randomAlphaOfLength(3), randomAlphaOfLength(4))
+                .build())
+            .transientSettings(Settings.builder()
+                .put("other_setting" + randomAlphaOfLength(3), randomAlphaOfLength(4))
+                .build())
+            .clusterUUID("uuid" + randomAlphaOfLength(3))
+            .clusterUUIDCommitted(randomBoolean())
+            .indexGraveyard(IndexGraveyardTests.createRandom())
+            .version(randomNonNegativeLong())
+            .put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance())
+            .build();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index 0338a64b6fe7e..b03c41484893f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
@@ -116,6 +117,11 @@ public void testSimpleJsonFromAndTo() throws IOException {
                         .putAlias(newAliasMetaDataBuilder("alias-bar1"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
+                .put("component_template", new ComponentTemplate(
+                    new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
+                        new CompressedXContent("{\"baz\":\"eggplant\"}"),
+                        Collections.singletonMap("alias", AliasMetaData.builder("alias").build())),
+                    5L, Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))))
                 .put(IndexMetaData.builder("test12")
                         .settings(settings(Version.CURRENT)
                                 .put("setting1", "value1")
@@ -285,6 +291,16 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").alias(), equalTo("alias-bar3"));
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").indexRouting(), equalTo("routing-bar"));
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").searchRouting(), equalTo("routing-bar"));
+
+        // component template
+        assertNotNull(parsedMetaData.componentTemplates().get("component_template"));
+        assertThat(parsedMetaData.componentTemplates().get("component_template").version(), is(5L));
+        assertThat(parsedMetaData.componentTemplates().get("component_template").metadata(),
+            equalTo(Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))));
+        assertThat(parsedMetaData.componentTemplates().get("component_template").template(),
+            equalTo(new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
+                new CompressedXContent("{\"baz\":\"eggplant\"}"),
+                Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
     }
 
     private static final String MAPPING_SOURCE1 = "{\"mapping1\":{\"text1\":{\"type\":\"string\"}}}";
diff --git a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
index 665315e171f5e..1105be8a7bf5a 100644
--- a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
@@ -19,12 +19,21 @@
 
 package org.elasticsearch.common.util;
 
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class BitArrayTests extends ESTestCase {
     public void testRandom() {
         try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
@@ -63,4 +72,31 @@ public void testTooBigIsNotSet() {
             }
         }
     }
+
+    public void testClearingDoesntAllocate() {
+        CircuitBreakerService breaker = mock(CircuitBreakerService.class);
+        ByteSizeValue max = new ByteSizeValue(1, ByteSizeUnit.KB);
+        when(breaker.getBreaker(CircuitBreaker.REQUEST)).thenReturn(new NoopCircuitBreaker(CircuitBreaker.REQUEST) {
+            private long total = 0;
+
+            @Override
+            public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
+                total += bytes;
+                if (total > max.getBytes()) {
+                    throw new CircuitBreakingException("test error", bytes, max.getBytes(), Durability.TRANSIENT);
+                }
+                return total;
+            }
+
+            @Override
+            public long addWithoutBreaking(long bytes) {
+                total += bytes;
+                return total;
+            }
+        });
+        BigArrays bigArrays = new BigArrays(null, breaker, CircuitBreaker.REQUEST, true);
+        try (BitArray bitArray = new BitArray(1, bigArrays)) {
+            bitArray.clear(100000000);
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
index ef7831658aa7c..edd74c824af44 100644
--- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
+++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.Index;
@@ -98,7 +99,7 @@ void syncFlush(String syncId) throws IOException {
             // make sure that we have committed translog; otherwise, we can flush after relaying translog in store recovery
             flush(true, true);
             // make sure that background merges won't happen; otherwise, IndexWriter#hasUncommittedChanges can become true again
-            forceMerge(false);
+            forceMerge(false, 1, false, false, false, UUIDs.randomBase64UUID());
             assertNotNull(indexWriter);
             try (ReleasableLock ignored = writeLock.acquire()) {
                 assertThat(getTranslogStats().getUncommittedOperations(), equalTo(0));
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index e587c9bc15b81..976f4d4cd4142 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -339,7 +339,7 @@ public void testSegmentsWithMergeFlag() throws Exception {
             engine.flush();
             final long gen1 = store.readLastCommittedSegmentsInfo().getGeneration();
             // now, optimize and wait for merges, see that we have no merge flag
-            engine.forceMerge(true);
+            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
 
             for (Segment segment : engine.segments(false)) {
                 assertThat(segment.getMergeId(), nullValue());
@@ -349,7 +349,7 @@ public void testSegmentsWithMergeFlag() throws Exception {
 
             final boolean flush = randomBoolean();
             final long gen2 = store.readLastCommittedSegmentsInfo().getGeneration();
-            engine.forceMerge(flush);
+            engine.forceMerge(flush, 1, false, false, false, UUIDs.randomBase64UUID());
             for (Segment segment : engine.segments(false)) {
                 assertThat(segment.getMergeId(), nullValue());
             }
@@ -1276,7 +1276,7 @@ public void testForceMergeWithSoftDeletesRetention() throws Exception {
             try (Engine.IndexCommitRef safeCommit = engine.acquireSafeIndexCommit()) {
                 safeCommitCheckpoint = Long.parseLong(safeCommit.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
             }
-            engine.forceMerge(true, 1, false, false, false);
+            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             Map<Long, Translog.Operation> ops = readAllOperationsInLucene(engine, mapperService)
                 .stream().collect(Collectors.toMap(Translog.Operation::seqNo, Function.identity()));
@@ -1300,7 +1300,7 @@ public void testForceMergeWithSoftDeletesRetention() throws Exception {
             globalCheckpoint.set(localCheckpoint);
             engine.syncTranslog();
 
-            engine.forceMerge(true, 1, false, false, false);
+            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             assertThat(readAllOperationsInLucene(engine, mapperService), hasSize(liveDocs.size()));
         }
@@ -1364,7 +1364,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
                     safeCommit.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
                 minSeqNoToRetain = Math.min(globalCheckpoint.get() + 1 - retainedExtraOps, safeCommitLocalCheckpoint + 1);
             }
-            engine.forceMerge(true, 1, false, false, false);
+            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             Map<Long, Translog.Operation> ops = readAllOperationsInLucene(engine, mapperService)
                 .stream().collect(Collectors.toMap(Translog.Operation::seqNo, Function.identity()));
@@ -1407,7 +1407,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
                 globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                 engine.syncTranslog();
             }
-            engine.forceMerge(true, 1, false, false, false);
+            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             assertThat(readAllOperationsInLucene(engine, mapperService), hasSize(liveDocsWithSource.size()));
         }
@@ -1443,7 +1443,8 @@ public void run() {
                                 engine.refresh("test");
                                 indexed.countDown();
                                 try {
-                                    engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean());
+                                    engine.forceMerge(
+                                        randomBoolean(), 1, false, randomBoolean(), randomBoolean(), UUIDs.randomBase64UUID());
                                 } catch (IOException e) {
                                     return;
                                 }
@@ -1460,7 +1461,7 @@ public void run() {
                 startGun.countDown();
                 int someIters = randomIntBetween(1, 10);
                 for (int i = 0; i < someIters; i++) {
-                    engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean());
+                    engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean(), UUIDs.randomBase64UUID());
                 }
                 indexed.await();
                 IOUtils.close(engine);
@@ -2838,7 +2839,7 @@ public void run() {
                     try {
                         switch (operation) {
                             case "optimize": {
-                                engine.forceMerge(true, 1, false, false, false);
+                                engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
                                 break;
                             }
                             case "refresh": {
@@ -4079,7 +4080,7 @@ public void testRandomOperations() throws Exception {
                 boolean onlyExpungeDeletes = randomBoolean();
                 int maxNumSegments = randomIntBetween(-1, 10);
                 try {
-                    engine.forceMerge(flush, maxNumSegments, onlyExpungeDeletes, false, false);
+                    engine.forceMerge(flush, maxNumSegments, onlyExpungeDeletes, false, false, UUIDs.randomBase64UUID());
                 } catch (IllegalArgumentException e) {
                     assertThat(e.getMessage(), containsString("only_expunge_deletes and max_num_segments are mutually exclusive"));
                     assertThat(onlyExpungeDeletes, is(true));
@@ -4767,7 +4768,7 @@ public void testShouldPeriodicallyFlushAfterMerge() throws Exception {
         engine.index(indexForDoc(doc));
         assertThat(engine.getTranslog().stats().getUncommittedOperations(), equalTo(2));
         engine.refresh("test");
-        engine.forceMerge(false, 1, false, false, false);
+        engine.forceMerge(false, 1, false, false, false, UUIDs.randomBase64UUID());
         assertBusy(() -> {
             // the merge listner runs concurrently after the force merge returned
             assertThat(engine.shouldPeriodicallyFlush(), equalTo(true));
@@ -5013,7 +5014,7 @@ private void assertOperationHistoryInLucene(List<Engine.Operation> operations) t
                     engine.flush();
                 }
                 if (rarely()) {
-                    engine.forceMerge(true);
+                    engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
                 }
             }
             MapperService mapperService = createMapperService();
@@ -5092,7 +5093,7 @@ public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException {
                     equalTo(engine.getMinRetainedSeqNo()));
             }
             if (rarely()) {
-                engine.forceMerge(randomBoolean());
+                engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
             }
             try (Closeable ignored = engine.acquireHistoryRetentionLock()) {
                 long minRetainSeqNos = engine.getMinRetainedSeqNo();
@@ -5410,7 +5411,7 @@ public long softUpdateDocuments(Term term, Iterable<? extends Iterable<? extends
                 assertThat(softDeletesEngine.getVersionMap().keySet(), empty());
                 softDeletesEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
                 if (randomBoolean()) {
-                    softDeletesEngine.forceMerge(randomBoolean());
+                    engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
                 }
                 assertThat(getDocIds(softDeletesEngine, true), equalTo(docs));
                 assertConsistentHistoryBetweenTranslogAndLuceneIndex(softDeletesEngine, createMapperService());
@@ -5613,7 +5614,7 @@ public void testPruneAwayDeletedButRetainedIds() throws Exception {
             for (int i = 0; i < numDocs; i++) {
                 index(engine, i);
             }
-            engine.forceMerge(true);
+            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
             engine.delete(new Engine.Delete("0", newUid("0"), primaryTerm.get()));
             engine.refresh("test");
             // now we have 2 segments since we now added a tombstone plus the old segment with the delete
@@ -5632,7 +5633,7 @@ public void testPruneAwayDeletedButRetainedIds() throws Exception {
             }
 
             // lets force merge the tombstone and the original segment and make sure the doc is still there but the ID term is gone
-            engine.forceMerge(true);
+            engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
             engine.refresh("test");
             try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                 IndexReader reader = searcher.getIndexReader();
@@ -5676,7 +5677,7 @@ public void testRecoverFromLocalTranslog() throws Exception {
                         engine.flush();
                     }
                     if (randomInt(100) < 5) {
-                        engine.forceMerge(randomBoolean(), 1, false, false, false);
+                        engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
                     }
                 }
                 if (randomBoolean()) {
diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
index b1eb9e6ded532..2583ef577ab36 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -101,10 +102,16 @@ public void testNoopAfterRegularEngine() throws IOException {
 
     public void testNoOpEngineStats() throws Exception {
         IOUtils.close(engine, store);
+        Settings.Builder settings = Settings.builder()
+            .put(defaultSettings.getSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0);
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
+            IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build());
+        
         final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
         try (Store store = createStore()) {
             Path translogPath = createTempDir();
-            EngineConfig config = config(defaultSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
+            EngineConfig config = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
             final int numDocs = scaledRandomIntBetween(10, 3000);
             int deletions = 0;
             try (InternalEngine engine = createEngine(config)) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index b276f7fc711af..d63a34ab501e3 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -202,7 +203,7 @@ private void testIgnoreMalfomedForValue(String value, String expectedException)
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(0, fields.length);
-        assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
+        assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
     }
 
     public void testChangeFormat() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 8e54a92a22228..86861901f7a07 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -154,11 +154,15 @@ public void testDotsWithExistingMapper() throws Exception {
             .endObject());
         ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", bytes, XContentType.JSON));
         assertNull(doc.dynamicMappingsUpdate()); // no update!
-        String[] values = doc.rootDoc().getValues("foo.bar.baz");
-        assertEquals(3, values.length);
-        assertEquals("123", values[0]);
-        assertEquals("456", values[1]);
-        assertEquals("789", values[2]);
+
+        IndexableField[] fields = doc.rootDoc().getFields("foo.bar.baz");
+        assertEquals(6, fields.length);
+        assertEquals(123, fields[0].numericValue());
+        assertEquals("123", fields[1].stringValue());
+        assertEquals(456, fields[2].numericValue());
+        assertEquals("456", fields[3].stringValue());
+        assertEquals(789, fields[4].numericValue());
+        assertEquals("789", fields[5].stringValue());
     }
 
     public void testDotsWithExistingNestedMapper() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
index 953a24ce00fa8..e37523ee730c0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.VersionUtils;
 
@@ -52,7 +53,7 @@ private static SortedSet<String> set(String... values) {
     }
 
     void assertFieldNames(Set<String> expected, ParsedDocument doc) {
-        String[] got = doc.rootDoc().getValues("_field_names");
+        String[] got = TermVectorsService.getValues(doc.rootDoc().getFields("_field_names"));
         assertEquals(expected, set(got));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index a34adaca2f144..37ea96e3df976 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -194,7 +195,7 @@ public void testIgnoreMalformed() throws Exception {
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(0, fields.length);
-        assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
+        assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
     }
 
     public void testNullValue() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index 0e25c0a6a212b..d5f0414708d64 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -128,6 +129,9 @@ public void testDefaults() throws Exception {
         fieldType = fields[1].fieldType();
         assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
         assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+
+        // used by TermVectorsService
+        assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field")));
     }
 
     public void testIgnoreAbove() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
index a1322c6614456..7adaf1b2a3233 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.randomizedtesting.annotations.Timeout;
-
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.Strings;
@@ -32,6 +31,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -250,7 +250,7 @@ public void testIgnoreMalformed() throws Exception {
 
                 IndexableField[] fields = doc.rootDoc().getFields("field");
                 assertEquals(0, fields.length);
-                assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
+                assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
index 79a93c04faa1f..0e00bf6677c8f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
@@ -21,14 +21,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.test.VersionUtils;
 import org.mockito.Mockito;
 
 public class TypeFieldTypeTests extends FieldTypeTestCase {
@@ -39,40 +32,14 @@ protected MappedFieldType createDefaultFieldType() {
 
     public void testTermsQuery() throws Exception {
         QueryShardContext context = Mockito.mock(QueryShardContext.class);
-        Version indexVersionCreated = VersionUtils.randomIndexCompatibleVersion(random());
-        Settings indexSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build();
-        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
-        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
-        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
-        Mockito.when(context.indexVersionCreated()).thenReturn(indexVersionCreated);
-
-        MapperService mapperService = Mockito.mock(MapperService.class);
-        Mockito.when(mapperService.documentMapper()).thenReturn(null);
-        Mockito.when(context.getMapperService()).thenReturn(mapperService);
 
         TypeFieldMapper.TypeFieldType ft = new TypeFieldMapper.TypeFieldType();
         ft.setName(TypeFieldMapper.NAME);
-        Query query = ft.termQuery("my_type", context);
-        assertEquals(new MatchNoDocsQuery(), query);
 
-        DocumentMapper mapper = Mockito.mock(DocumentMapper.class);
-        Mockito.when(mapper.type()).thenReturn("my_type");
-        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
-        query = ft.termQuery("my_type", context);
+        Query query = ft.termQuery("_doc", context);
         assertEquals(new MatchAllDocsQuery(), query);
 
-        Mockito.when(mapperService.hasNested()).thenReturn(true);
-        query = ft.termQuery("my_type", context);
-        assertEquals(Queries.newNonNestedFilter(), query);
-
-        mapper = Mockito.mock(DocumentMapper.class);
-        Mockito.when(mapper.type()).thenReturn("other_type");
-        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
-        query = ft.termQuery("my_type", context);
+        query = ft.termQuery("other_type", context);
         assertEquals(new MatchNoDocsQuery(), query);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
index 8e736b12d6a1f..ed6b58390f645 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.XIntervals;
 import org.apache.lucene.queries.intervals.IntervalQuery;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
@@ -449,7 +448,7 @@ public void testPrefixes() throws IOException {
         String json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
             "\"prefix\" : { \"prefix\" : \"term\" } } } }";
         IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
-        Query expected = new IntervalQuery(STRING_FIELD_NAME, XIntervals.prefix(new BytesRef("term")));
+        Query expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.prefix(new BytesRef("term")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String no_positions_json = "{ \"intervals\" : { \"" + NO_POSITIONS_FIELD + "\": { " +
@@ -476,7 +475,7 @@ public void testPrefixes() throws IOException {
             "\"prefix\" : { \"prefix\" : \"t\" } } } }";
         builder = (IntervalQueryBuilder) parseQuery(short_prefix_json);
         expected = new IntervalQuery(PREFIXED_FIELD, Intervals.or(
-            Intervals.fixField(PREFIXED_FIELD + "._index_prefix", XIntervals.wildcard(new BytesRef("t?"))),
+            Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.wildcard(new BytesRef("t?"))),
             Intervals.term("t")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
@@ -508,7 +507,7 @@ public void testWildcard() throws IOException {
             "\"wildcard\" : { \"pattern\" : \"Te?m\" } } } }";
 
         IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
-        Query expected = new IntervalQuery(STRING_FIELD_NAME, XIntervals.wildcard(new BytesRef("te?m")));
+        Query expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.wildcard(new BytesRef("te?m")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String no_positions_json = "{ \"intervals\" : { \"" + NO_POSITIONS_FIELD + "\": { " +
@@ -522,14 +521,14 @@ public void testWildcard() throws IOException {
             "\"wildcard\" : { \"pattern\" : \"Te?m\", \"analyzer\" : \"keyword\" } } } }";
 
         builder = (IntervalQueryBuilder) parseQuery(keyword_json);
-        expected = new IntervalQuery(STRING_FIELD_NAME, XIntervals.wildcard(new BytesRef("Te?m")));
+        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.wildcard(new BytesRef("Te?m")));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String fixed_field_json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
             "\"wildcard\" : { \"pattern\" : \"Te?m\", \"use_field\" : \"masked_field\" } } } }";
 
         builder = (IntervalQueryBuilder) parseQuery(fixed_field_json);
-        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD, XIntervals.wildcard(new BytesRef("te?m"))));
+        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("te?m"))));
         assertEquals(expected, builder.toQuery(createShardContext()));
 
         String fixed_field_json_no_positions = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
@@ -544,14 +543,14 @@ public void testWildcard() throws IOException {
 
         builder = (IntervalQueryBuilder) parseQuery(fixed_field_analyzer_json);
         expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD,
-            XIntervals.wildcard(new BytesRef("Te?m"))));
+            Intervals.wildcard(new BytesRef("Te?m"))));
         assertEquals(expected, builder.toQuery(createShardContext()));
     }
 
     private static IntervalsSource buildFuzzySource(String term, String label, int prefixLength, boolean transpositions, int editDistance) {
         FuzzyQuery fq = new FuzzyQuery(new Term("field", term), editDistance, prefixLength, 128, transpositions);
         CompiledAutomaton[] automata = fq.getAutomata();
-        return XIntervals.multiterm(automata[automata.length - 1], label);
+        return Intervals.multiterm(automata[automata.length - 1], label);
     }
 
     public void testFuzzy() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index c9c43a1e90186..852c64fd6fdfd 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -556,13 +556,6 @@ public void testParseRelation() {
         assertEquals(ShapeRelation.INTERSECTS, builder.relation());
     }
 
-    public void testTypeField() throws IOException {
-        RangeQueryBuilder builder = QueryBuilders.rangeQuery("_type")
-            .from("value1");
-        builder.doToQuery(createShardContext());
-        assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
-    }
-
     /**
      * Range queries should generally be cacheable, at least the ones we create randomly.
      * This test makes sure we also test the non-cacheable cases regularly.
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 9482e0bb87d49..3eff0d1f819a5 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -75,7 +76,9 @@ protected void doAssertLuceneQuery(WildcardQueryBuilder queryBuilder, Query quer
 
             assertThat(wildcardQuery.getField(), equalTo(expectedFieldName));
             assertThat(wildcardQuery.getTerm().field(), equalTo(expectedFieldName));
-            assertThat(wildcardQuery.getTerm().text(), equalTo(queryBuilder.value()));
+            // wildcard queries get normalized
+            String text = wildcardQuery.getTerm().text().toLowerCase(Locale.ROOT);
+            assertThat(text, equalTo(text));
         } else {
             Query expected = new MatchNoDocsQuery("unknown field [" + expectedFieldName + "]");
             assertEquals(expected, query);
@@ -138,14 +141,14 @@ public void testTypeField() throws IOException {
         builder.doToQuery(createShardContext());
         assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
     }
-    
+
     public void testRewriteIndexQueryToMatchNone() throws IOException {
         WildcardQueryBuilder query = new WildcardQueryBuilder("_index", "does_not_exist");
         QueryShardContext queryShardContext = createShardContext();
         QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
-    }   
-    
+    }
+
     public void testRewriteIndexQueryNotMatchNone() throws IOException {
         String fullIndexName = getIndex().getName();
         String firstHalfOfIndexName = fullIndexName.substring(0,fullIndexName.length()/2);
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 0e7bf966c2d0a..9401d9dd96d50 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -4104,4 +4104,22 @@ public void testCloseShardWhileEngineIsWarming() throws Exception {
         recoveryThread.join();
         shard.store().close();
     }
+
+    public void testRecordsForceMerges() throws IOException {
+        IndexShard shard = newStartedShard(true);
+        final String initialForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
+        assertThat(initialForceMergeUUID, nullValue());
+        final ForceMergeRequest firstForceMergeRequest = new ForceMergeRequest().maxNumSegments(1);
+        shard.forceMerge(firstForceMergeRequest);
+        final String secondForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
+        assertThat(secondForceMergeUUID, notNullValue());
+        assertThat(secondForceMergeUUID, equalTo(firstForceMergeRequest.forceMergeUUID()));
+        final ForceMergeRequest secondForceMergeRequest = new ForceMergeRequest().maxNumSegments(1);
+        shard.forceMerge(secondForceMergeRequest);
+        final String thirdForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
+        assertThat(thirdForceMergeUUID, notNullValue());
+        assertThat(thirdForceMergeUUID, not(equalTo(secondForceMergeUUID)));
+        assertThat(thirdForceMergeUUID, equalTo(secondForceMergeRequest.forceMergeUUID()));
+        closeShards(shard);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 560f052664604..0a68c54e75b47 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -51,6 +52,7 @@
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ScrollContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -118,8 +120,8 @@ public void testPreProcess() throws Exception {
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
-            DefaultSearchContext context1 = new DefaultSearchContext(1L, shardSearchRequest, target, searcher, null, indexService,
-                indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context1 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 1L),
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
             context1.from(300);
 
             // resultWindow greater than maxResultWindow and scrollContext is null
@@ -159,8 +161,8 @@ public void testPreProcess() throws Exception {
                 + "] index level setting."));
 
             // rescore is null but sliceBuilder is not null
-            DefaultSearchContext context2 = new DefaultSearchContext(2L, shardSearchRequest, target, searcher,
-                null, indexService, indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context2 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 2L),
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
             int numSlices = maxSlicesPerScroll + randomIntBetween(1, 100);
@@ -176,8 +178,8 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.getAliasFilter()).thenReturn(AliasFilter.EMPTY);
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
-            DefaultSearchContext context3 = new DefaultSearchContext(3L, shardSearchRequest, target, searcher, null,
-                indexService, indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context3 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 3L),
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
             assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query()));
@@ -186,8 +188,8 @@ public void testPreProcess() throws Exception {
             when(queryShardContext.fieldMapper(anyString())).thenReturn(mock(MappedFieldType.class));
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
-            DefaultSearchContext context4 = new DefaultSearchContext(4L, shardSearchRequest, target, searcher, null,
-                indexService, indexShard, bigArrays, null, timeout, null);
+            DefaultSearchContext context4 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 4L),
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
             context4.sliceBuilder(new SliceBuilder(0,2)).parsedQuery(parsedQuery).preProcess(false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 2e61d63c81117..7607d519d1d8c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -77,6 +78,7 @@
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -84,6 +86,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
@@ -321,12 +324,12 @@ public void onFailure(Exception e) {
                         SearchPhaseResult searchPhaseResult = result.get();
                         IntArrayList intCursors = new IntArrayList(1);
                         intCursors.add(0);
-                        ShardFetchRequest req = new ShardFetchRequest(searchPhaseResult.getRequestId(), intCursors, null/* not a scroll */);
+                        ShardFetchRequest req = new ShardFetchRequest(searchPhaseResult.getContextId(), intCursors, null/* not a scroll */);
                         PlainActionFuture<FetchSearchResult> listener = new PlainActionFuture<>();
                         service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), listener);
                         listener.get();
                         if (useScroll) {
-                            service.freeContext(searchPhaseResult.getRequestId());
+                            service.freeContext(searchPhaseResult.getContextId());
                         }
                     } catch (ExecutionException ex) {
                         assertThat(ex.getCause(), instanceOf(RuntimeException.class));
@@ -536,6 +539,47 @@ public void testMaxOpenScrollContexts() throws RuntimeException, IOException {
             ex.getMessage());
     }
 
+    public void testOpenScrollContextsConcurrently() throws Exception {
+        createIndex("index");
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexShard indexShard = indicesService.indexServiceSafe(resolveIndex("index")).getShard(0);
+
+        final int maxScrollContexts = SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY);
+        final SearchService searchService = getInstanceFromNode(SearchService.class);
+        Thread[] threads = new Thread[randomIntBetween(2, 8)];
+        CountDownLatch latch = new CountDownLatch(threads.length);
+        for (int i = 0; i < threads.length; i++) {
+            threads[i] = new Thread(() -> {
+                latch.countDown();
+                try {
+                    latch.await();
+                    for (; ; ) {
+                        SearchService.SearchRewriteContext rewriteContext =
+                            searchService.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
+                        try {
+                            searchService.createAndPutContext(rewriteContext);
+                        } catch (ElasticsearchException e) {
+                            assertThat(e.getMessage(), equalTo(
+                                "Trying to create too many scroll contexts. Must be less than or equal to: " +
+                                    "[" + maxScrollContexts + "]. " +
+                                    "This limit can be set by changing the [search.max_open_scroll_context] setting."));
+                            return;
+                        }
+                    }
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+            });
+            threads[i].setName("elasticsearch[node_s_0][search]");
+            threads[i].start();
+        }
+        for (Thread thread : threads) {
+            thread.join();
+        }
+        assertThat(searchService.getActiveContexts(), equalTo(maxScrollContexts));
+        searchService.freeAllScrollContexts();
+    }
+
     public static class FailOnRewriteQueryPlugin extends Plugin implements SearchPlugin {
         @Override
         public List<QuerySpec<?>> getQueries() {
@@ -929,4 +973,40 @@ public void testDeleteIndexWhileSearch() throws Exception {
             searcher.join();
         }
     }
+
+    public void testLookUpSearchContext() throws Exception {
+        createIndex("index");
+        SearchService searchService = getInstanceFromNode(SearchService.class);
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        IndexShard indexShard = indexService.getShard(0);
+        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(
+            OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
+            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
+        List<SearchContextId> contextIds = new ArrayList<>();
+        int numContexts = randomIntBetween(1, 10);
+        for (int i = 0; i < numContexts; i++) {
+            SearchService.SearchRewriteContext rewriteContext = searchService.acquireSearcherAndRewrite(shardSearchRequest, indexShard);
+            final SearchContext searchContext = searchService.createContext(rewriteContext);
+            assertThat(searchContext.id().getId(), equalTo((long) (i + 1)));
+            searchService.putContext(searchContext);
+            contextIds.add(searchContext.id());
+        }
+        assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+        while (contextIds.isEmpty() == false) {
+            final SearchContextId contextId = randomFrom(contextIds);
+            assertFalse(searchService.freeContext(new SearchContextId(UUIDs.randomBase64UUID(), contextId.getId())));
+            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+            if (randomBoolean()) {
+                assertTrue(searchService.freeContext(contextId));
+            } else {
+                assertTrue(searchService.freeContext((new SearchContextId("", contextId.getId()))));
+            }
+            contextIds.remove(contextId);
+            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+            assertFalse(searchService.freeContext(new SearchContextId("", contextId.getId())));
+            assertFalse(searchService.freeContext(contextId));
+            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
deleted file mode 100644
index ddd8c1920b99e..0000000000000
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.aggregations.bucket.missing.Missing;
-import org.elasticsearch.search.aggregations.metrics.Avg;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.hamcrest.Matchers;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.missing;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.core.IsNull.notNullValue;
-
-@ESIntegTestCase.SuiteScopeTestCase
-public class MissingIT extends ESIntegTestCase {
-
-    static int numDocs, numDocsMissing, numDocsUnmapped;
-
-    @Override
-    public void setupSuiteScopeCluster() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("idx")
-                .setMapping("tag", "type=keyword").get());
-        List<IndexRequestBuilder> builders = new ArrayList<>();
-        numDocs = randomIntBetween(5, 20);
-        numDocsMissing = randomIntBetween(1, numDocs - 1);
-        for (int i = 0; i < numDocsMissing; i++) {
-            builders.add(client().prepareIndex("idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("value", i)
-                    .endObject()));
-        }
-        for (int i = numDocsMissing; i < numDocs; i++) {
-            builders.add(client().prepareIndex("idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("tag", "tag1")
-                    .endObject()));
-        }
-
-        createIndex("unmapped_idx");
-        numDocsUnmapped = randomIntBetween(2, 5);
-        for (int i = 0; i < numDocsUnmapped; i++) {
-            builders.add(client().prepareIndex("unmapped_idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("value", i)
-                    .endObject()));
-        }
-
-        prepareCreate("empty_bucket_idx").setMapping("value", "type=integer").get();
-        for (int i = 0; i < 2; i++) {
-            builders.add(client().prepareIndex("empty_bucket_idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("value", i*2)
-                    .endObject()));
-        }
-
-        indexRandom(true, builders);
-        ensureSearchable();
-    }
-
-    public void testUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("unmapped_idx")
-                .addAggregation(missing("missing_tag").field("tag"))
-                .get();
-
-        assertSearchResponse(response);
-
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsUnmapped));
-    }
-
-    public void testPartiallyUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx", "unmapped_idx")
-                .addAggregation(missing("missing_tag").field("tag"))
-                .get();
-
-        assertSearchResponse(response);
-
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped));
-    }
-
-    public void testSimple() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(missing("missing_tag").field("tag"))
-                .get();
-
-        assertSearchResponse(response);
-
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing));
-    }
-
-    public void testWithSubAggregation() throws Exception {
-        SearchResponse response = client().prepareSearch("idx", "unmapped_idx")
-                .addAggregation(missing("missing_tag").field("tag")
-                        .subAggregation(avg("avg_value").field("value")))
-                .get();
-
-        assertSearchResponse(response);
-
-        assertThat("Not all shards are initialized", response.getSuccessfulShards(), equalTo(response.getTotalShards()));
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped));
-        assertThat((long) ((InternalAggregation)missing).getProperty("_count"), equalTo((long) numDocsMissing + numDocsUnmapped));
-        assertThat(missing.getAggregations().asList().isEmpty(), is(false));
-
-        long sum = 0;
-        for (int i = 0; i < numDocsMissing; ++i) {
-            sum += i;
-        }
-        for (int i = 0; i < numDocsUnmapped; ++i) {
-            sum += i;
-        }
-        Avg avgValue = missing.getAggregations().get("avg_value");
-        assertThat(avgValue, notNullValue());
-        assertThat(avgValue.getName(), equalTo("avg_value"));
-        assertThat(avgValue.getValue(), equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
-        assertThat((double) ((InternalAggregation)missing).getProperty("avg_value.value"),
-                equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
-    }
-
-    public void testEmptyAggregation() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0)
-                        .subAggregation(missing("missing").field("value")))
-                .get();
-
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
-        Histogram histo = searchResponse.getAggregations().get("histo");
-        assertThat(histo, Matchers.notNullValue());
-        Histogram.Bucket bucket = histo.getBuckets().get(1);
-        assertThat(bucket, Matchers.notNullValue());
-
-        Missing missing = bucket.getAggregations().get("missing");
-        assertThat(missing, Matchers.notNullValue());
-        assertThat(missing.getName(), equalTo("missing"));
-        assertThat(missing.getDocCount(), is(0L));
-    }
-
-
-}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
index 7685cd9067ddc..dbea1d7f2729e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
@@ -2077,40 +2077,52 @@ private static long asLong(String dateTime) {
 
     private static Sort buildIndexSort(List<CompositeValuesSourceBuilder<?>> sources, Map<String, MappedFieldType> fieldTypes) {
         List<SortField> sortFields = new ArrayList<>();
+        Map<String, MappedFieldType> remainingFieldTypes = new HashMap<>(fieldTypes);
         for (CompositeValuesSourceBuilder<?> source : sources) {
-            MappedFieldType type = fieldTypes.get(source.field());
-            if (type instanceof KeywordFieldMapper.KeywordFieldType) {
-                sortFields.add(new SortedSetSortField(type.name(), false));
-            } else if (type instanceof DateFieldMapper.DateFieldType) {
-                sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.LONG, false));
-            } else if (type instanceof NumberFieldMapper.NumberFieldType) {
-                boolean comp = false;
-                switch (type.typeName()) {
-                    case "byte":
-                    case "short":
-                    case "integer":
-                        comp = true;
-                        sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.INT, false));
-                        break;
-
-                    case "long":
-                        sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.LONG, false));
-                        break;
-
-                    case "float":
-                    case "double":
-                        comp = true;
-                        sortFields.add(new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false));
-                        break;
-
-                    default:
-                        break;
-                }
-                if (comp == false) {
-                    break;
-                }
+            MappedFieldType type = fieldTypes.remove(source.field());
+            remainingFieldTypes.remove(source.field());
+            SortField sortField = sortFieldFrom(type);
+            if (sortField == null) {
+                break;
+            }
+            sortFields.add(sortField);
+            if (sortField instanceof SortedNumericSortField && ((SortedNumericSortField) sortField).getType() == SortField.Type.LONG) {
+                break;
+            }
+        }
+        while (remainingFieldTypes.size() > 0 && randomBoolean()) {
+            // Add extra unused sorts
+            List<String> fields = new ArrayList<>(remainingFieldTypes.keySet());
+            Collections.sort(fields);
+            String field = fields.get(between(0, fields.size() - 1));
+            SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field));
+            if (sortField != null) {
+                sortFields.add(sortField);
+            }
+        }
+        return sortFields.size() > 0 ? new Sort(sortFields.toArray(SortField[]::new)) : null;
+    }
+    
+    private static SortField sortFieldFrom(MappedFieldType type) {
+        if (type instanceof KeywordFieldMapper.KeywordFieldType) {
+            return new SortedSetSortField(type.name(), false);
+        } else if (type instanceof DateFieldMapper.DateFieldType) {
+            return new SortedNumericSortField(type.name(), SortField.Type.LONG, false);
+        } else if (type instanceof NumberFieldMapper.NumberFieldType) {
+            switch (type.typeName()) {
+                case "byte":
+                case "short":
+                case "integer":
+                    return new SortedNumericSortField(type.name(), SortField.Type.INT, false);
+                case "long":
+                    return new SortedNumericSortField(type.name(), SortField.Type.LONG, false);
+                case "float":
+                case "double":
+                    return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false);
+                default:
+                    return null;
             }
         }
-        return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null;
+        return null;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java
index 2059f465c3756..7657754b85e9e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java
@@ -33,9 +33,12 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -70,6 +73,16 @@ public abstract class GeoGridAggregatorTestCase<T extends InternalGeoGridBucket>
      */
     protected abstract GeoGridAggregationBuilder createBuilder(String name);
 
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return createBuilder("foo").field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(CoreValuesSourceType.GEOPOINT);
+    }
+
     public void testNoDocs() throws IOException {
         testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), null, geoGrid -> {
             assertEquals(0, geoGrid.getBuckets().size());
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
index dbc3ac3b490c2..3665fcded7280 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
@@ -20,174 +20,617 @@
 package org.elasticsearch.search.aggregations.bucket.missing;
 
 import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
-import org.elasticsearch.common.lucene.search.Queries;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.mapper.RangeType;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
+import org.elasticsearch.search.aggregations.metrics.InternalSum;
+import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.common.lucene.search.Queries.newMatchAllQuery;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
+import static org.hamcrest.collection.IsEmptyCollection.empty;
 
 public class MissingAggregatorTests extends AggregatorTestCase {
+
+    private static final String VALUE_SCRIPT_PARAMS = "value_script_params";
+    private static final String VALUE_SCRIPT = "value_script";
+    private static final String FIELD_SCRIPT_PARAMS = "field_script_params";
+    private static final String FIELD_SCRIPT = "field_script";
+    private static final long DEFAULT_INC_PARAM = 1;
+    private static final long DEFAULT_THRESHOLD_PARAM = 50;
+
     public void testMatchNoDocs() throws IOException {
-        int numDocs = randomIntBetween(10, 200);
-        testBothCases(numDocs,
-            "field",
-            Queries.newMatchAllQuery(),
-            doc -> doc.add(new SortedNumericDocValuesField("field", randomLong())),
-            internalMissing -> {
-                assertEquals(internalMissing.getDocCount(), 0);
-                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
-            });
+        final int numDocs = randomIntBetween(10, 200);
+
+        final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        fieldType.setName("field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(fieldType.name());
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> {
+                for (int i = 0; i < numDocs; i++) {
+                    writer.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong())));
+                }
+            },
+            (InternalMissing missing) -> {
+                assertEquals(0, missing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(missing));
+            },
+            singleton(fieldType)
+        );
     }
 
     public void testMatchAllDocs() throws IOException {
         int numDocs = randomIntBetween(10, 200);
-        testBothCases(numDocs,
-            "field",
-            Queries.newMatchAllQuery(),
-            doc -> doc.add(new SortedNumericDocValuesField("another_field", randomLong())),
-            internalMissing -> {
-                assertEquals(internalMissing.getDocCount(), numDocs);
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
-            });
+
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        anotherFieldType.setName("another_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(aggFieldType.name());
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> {
+                for (int i = 0; i < numDocs; i++) {
+                    writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
+                }
+            },
+            (InternalMissing missing) -> {
+                assertEquals(numDocs, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
+            },
+            List.of(aggFieldType, anotherFieldType)
+        );
     }
 
     public void testMatchSparse() throws IOException {
-        int numDocs = randomIntBetween(100, 200);
-        final AtomicInteger count = new AtomicInteger();
-        testBothCases(numDocs,
-            "field",
-            Queries.newMatchAllQuery(),
-            doc -> {
-                if (randomBoolean()) {
-                    doc.add(new SortedNumericDocValuesField("another_field", randomLong()));
-                    count.incrementAndGet();
-                } else {
-                    doc.add(new SortedNumericDocValuesField("field", randomLong()));
-                }
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        anotherFieldType.setName("another_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(aggFieldType.name());
+
+        final int numDocs = randomIntBetween(100, 200);
+        int docsMissingAggField = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                docs.add(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
+            } else {
+                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
+                docsMissingAggField++;
+            }
+        }
+        final int finalDocsMissingAggField = docsMissingAggField;
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingAggField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
-            internalMissing -> {
-                assertEquals(internalMissing.getDocCount(), count.get());
-                count.set(0);
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
-            });
+            List.of(aggFieldType, anotherFieldType)
+        );
     }
 
     public void testMatchSparseRangeField() throws IOException {
-        int numDocs = randomIntBetween(100, 200);
-        final AtomicInteger count = new AtomicInteger();
-        final String fieldName = "field";
-        RangeType rangeType = RangeType.DOUBLE;
-        final BinaryDocValuesField field = new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(
-            new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true))));
-        MappedFieldType fieldType = new RangeFieldMapper.Builder(fieldName, rangeType).fieldType();
-        fieldType.setName(fieldName);
-        testBothCases(numDocs,
-            fieldName,
-            Queries.newMatchAllQuery(),
-            doc -> {
-                if (randomBoolean()) {
-                    doc.add(new SortedNumericDocValuesField("another_field", randomLong()));
-                    count.incrementAndGet();
-                } else {
-                    doc.add(field);
-                }
+        final RangeType rangeType = RangeType.DOUBLE;
+        MappedFieldType aggFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType();
+        aggFieldType.setName("agg_field");
+        final MappedFieldType anotherFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType();
+        anotherFieldType.setName("another_field");
+
+        final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true);
+        final BytesRef encodedRange = rangeType.encodeRanges(singleton(range));
+        final BinaryDocValuesField encodedRangeField = new BinaryDocValuesField(aggFieldType.name(), encodedRange);
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(aggFieldType.name());
+
+        final int numDocs = randomIntBetween(100, 200);
+        int docsMissingAggField = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                docs.add(singleton(encodedRangeField));
+            } else {
+                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
+                docsMissingAggField++;
+            }
+        }
+        final int finalDocsMissingAggField = docsMissingAggField;
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingAggField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
-            internalMissing -> {
-                assertEquals(internalMissing.getDocCount(), count.get());
-                count.set(0);
+            List.of(aggFieldType, anotherFieldType)
+        );
+    }
+
+    public void testSubAggregation() throws IOException {
+        final MappedFieldType missingAggFieldType = new KeywordFieldMapper.Builder("_name").fieldType();
+        missingAggFieldType.setName("missing_agg_field");
+        missingAggFieldType.setHasDocValues(true);
+        final MappedFieldType sumAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        sumAggFieldType.setName("sum_agg_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("missing", null)
+            .field(missingAggFieldType.name())
+            .subAggregation(
+              new SumAggregationBuilder("sum")
+                .field(sumAggFieldType.name())
+            );
+
+        final int numDocs = randomIntBetween(10, 20);
+        int docsMissingAggField = 0;
+        long expectedSum = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            final Set<IndexableField> doc = new HashSet<>();
+            final long sumFieldValue = randomLongBetween(0, 1000);
+            doc.add(new SortedNumericDocValuesField(sumAggFieldType.name(), sumFieldValue));
+            if (randomBoolean()) {
+                doc.add(new SortedDocValuesField(missingAggFieldType.name(), new BytesRef(randomUnicodeOfLengthBetween(2, 20))));
+            } else {
+                docsMissingAggField++;
+                expectedSum += sumFieldValue;
+            }
+            docs.add(doc);
+        }
+        final int finalDocsMissingAggField = docsMissingAggField;
+        final long finalExpectedSum = expectedSum;
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            (InternalMissing internalMissing) -> {
+                assertEquals(finalDocsMissingAggField, internalMissing.getDocCount());
                 assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
-            }, fieldType);
+
+                assertThat(internalMissing.getAggregations().asList(), not(empty()));
+                final InternalSum internalSum = internalMissing.getAggregations().get("sum");
+                assertEquals(finalExpectedSum, internalSum.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(internalSum));
+            },
+            List.of(missingAggFieldType, sumAggFieldType)
+        );
     }
 
+    public void testEmptyBucket() throws IOException {
+        final MappedFieldType histoAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        histoAggFieldType.setName("histo_agg_field");
+        final MappedFieldType missingAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        missingAggFieldType.setName("missing_agg_field");
 
-    public void testMissingField() throws IOException {
-        int numDocs = randomIntBetween(10, 20);
-        testBothCases(numDocs,
-            "unknown_field",
-            Queries.newMatchAllQuery(),
-            doc -> {
-                doc.add(new SortedNumericDocValuesField("field", randomLong()));
+        final HistogramAggregationBuilder builder = new HistogramAggregationBuilder("histo")
+            .field(histoAggFieldType.name())
+            .interval(1)
+            .minDocCount(0)
+            .subAggregation(
+              new MissingAggregationBuilder("missing", null)
+                .field(missingAggFieldType.name())
+            );
+
+        testCaseWithReduce(
+            newMatchAllQuery(),
+            builder,
+            writer -> {
+                writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 0)));
+                writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 2)));
             },
-            internalMissing -> {
-                assertEquals(internalMissing.getDocCount(), numDocs);
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
-            });
-    }
-
-    private void testBothCases(int numDocs,
-                               String fieldName,
-                               Query query,
-                               Consumer<Document> consumer,
-                               Consumer<InternalMissing> verify) throws IOException {
-        NumberFieldMapper.Builder mapperBuilder = new NumberFieldMapper.Builder("_name",
-            NumberFieldMapper.NumberType.LONG);
-        final MappedFieldType fieldType = mapperBuilder.fieldType();
-        fieldType.setHasDocValues(true);
-        fieldType.setName(fieldName);
-        testBothCases(numDocs, fieldName, query, consumer, verify, fieldType);
-    }
-
-    private void testBothCases(int numDocs,
-                               String fieldName,
-                               Query query,
-                               Consumer<Document> consumer,
-                               Consumer<InternalMissing> verify,
-                               MappedFieldType fieldType) throws IOException {
-        executeTestCase(numDocs, fieldName, query, consumer, verify, false, fieldType);
-        executeTestCase(numDocs, fieldName, query, consumer, verify, true, fieldType);
-
-    }
-
-    private void executeTestCase(int numDocs,
-                                 String fieldName,
-                                 Query query,
-                                 Consumer<Document> consumer,
-                                 Consumer<InternalMissing> verify,
-                                 boolean reduced,
-                                 MappedFieldType fieldType) throws IOException {
-        try (Directory directory = newDirectory()) {
-            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
-                Document document = new Document();
+            (InternalHistogram histogram) -> {
+                assertThat(histogram.getBuckets(), hasSize(3));
+
+                {
+                    assertThat(histogram.getBuckets().get(0), notNullValue());
+                    final InternalMissing missing = histogram.getBuckets().get(0).getAggregations().get("missing");
+                    assertEquals(1, missing.getDocCount());
+                    assertTrue(AggregationInspectionHelper.hasValue(missing));
+                }
+                {
+                    assertThat(histogram.getBuckets().get(1), notNullValue());
+                    final InternalMissing missing = histogram.getBuckets().get(1).getAggregations().get("missing");
+                    assertEquals(0, missing.getDocCount());
+                    assertFalse(AggregationInspectionHelper.hasValue(missing));
+                }
+                {
+                    assertThat(histogram.getBuckets().get(2), notNullValue());
+                    final InternalMissing missing = histogram.getBuckets().get(2).getAggregations().get("missing");
+                    assertEquals(1, missing.getDocCount());
+                    assertTrue(AggregationInspectionHelper.hasValue(missing));
+                }
+            },
+            List.of(histoAggFieldType, missingAggFieldType),
+            true
+        );
+    }
+
+    public void testUnmappedWithoutMissingParam() throws IOException {
+        final int numDocs = randomIntBetween(10, 20);
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field("unknown_field");
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> {
                 for (int i = 0; i < numDocs; i++) {
-                    if (frequently()) {
-                        indexWriter.commit();
-                    }
-                    consumer.accept(document);
-                    indexWriter.addDocument(document);
-                    document.clear();
+                    writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
+                }
+            },
+            (InternalMissing missing) -> {
+                assertEquals(numDocs, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
+            },
+            singleton(aggFieldType)
+        );
+    }
+
+    public void testUnmappedWithMissingParam() throws IOException {
+        final int numDocs = randomIntBetween(10, 20);
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field("unknown_field")
+            .missing(randomLong());
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> {
+                for (int i = 0; i < numDocs; i++) {
+                    writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
+                }
+            },
+            (InternalMissing missing) -> {
+                assertEquals(0, missing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(missing));
+            },
+            singleton(aggFieldType)
+        );
+    }
+
+    public void testSingleValuedFieldPartiallyUnmapped() throws IOException {
+        final int numDocs = randomIntBetween(10, 20);
+
+        final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        fieldType.setName("field");
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(fieldType.name());
+
+        try (Directory mappedDirectory = newDirectory(); Directory unmappedDirectory = newDirectory()) {
+            try (RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory)) {
+                for (int i = 0; i < numDocs; i++) {
+                    mappedWriter.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong())));
                 }
             }
 
-            try (IndexReader indexReader = DirectoryReader.open(directory)) {
-                IndexSearcher indexSearcher =
-                    newSearcher(indexReader, true, true);
-                MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null);
-                builder.field(fieldName);
+            new RandomIndexWriter(random(), unmappedDirectory).close();
+
+            try (IndexReader mappedReader = DirectoryReader.open(mappedDirectory);
+                 IndexReader unmappedReader = DirectoryReader.open(unmappedDirectory);
+                 MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) {
+
+                final IndexSearcher searcher = newSearcher(multiReader, true, true);
+
+                final InternalMissing internalMissing = searchAndReduce(searcher, newMatchAllQuery(), builder, fieldType);
+                assertEquals(0, internalMissing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
+            }
+        }
+    }
+
+    public void testMissingParam() throws IOException {
+        final int numDocs = randomIntBetween(10, 20);
+
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        anotherFieldType.setName("another_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(aggFieldType.name())
+            .missing(randomLong());
 
-                InternalMissing missing;
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> {
+                for (int i = 0; i < numDocs; i++) {
+                    writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
+                }
+            },
+            (InternalMissing missing) -> {
+                assertEquals(0, missing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(missing));
+            },
+            List.of(aggFieldType, anotherFieldType)
+        );
+    }
+
+    public void testMultiValuedField() throws IOException {
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        anotherFieldType.setName("another_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(aggFieldType.name());
+
+        final int numDocs = randomIntBetween(100, 200);
+        int docsMissingAggField = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                final long randomLong = randomLong();
+                docs.add(Set.of(
+                    new SortedNumericDocValuesField(aggFieldType.name(), randomLong),
+                    new SortedNumericDocValuesField(aggFieldType.name(), randomLong + 1)
+                ));
+            } else {
+                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
+                docsMissingAggField++;
+            }
+        }
+        final int finalDocsMissingAggField = docsMissingAggField;
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingAggField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
+            },
+            List.of(aggFieldType, anotherFieldType)
+        );
+    }
+
+    public void testSingleValuedFieldWithValueScript() throws IOException {
+        valueScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, emptyMap()));
+    }
+
+    public void testSingleValuedFieldWithValueScriptWithParams() throws IOException {
+        valueScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_PARAMS, singletonMap("inc", 10)));
+    }
+
+    private void valueScriptTestCase(Script script) throws IOException {
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+        final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        anotherFieldType.setName("another_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(aggFieldType.name())
+            .script(script);
+
+        final int numDocs = randomIntBetween(100, 200);
+        int docsMissingAggField = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                docs.add(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
+            } else {
+                docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
+                docsMissingAggField++;
+            }
+        }
+        final int finalDocsMissingField = docsMissingAggField;
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
+            },
+            List.of(aggFieldType, anotherFieldType)
+        );
+    }
+
+    public void testMultiValuedFieldWithFieldScriptWithParams() throws IOException {
+        final long threshold = 10;
+        final Map<String, Object> params = Map.of("field", "agg_field", "threshold", threshold);
+        fieldScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_PARAMS, params), threshold);
+    }
+
+    public void testMultiValuedFieldWithFieldScript() throws IOException {
+        fieldScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT, singletonMap("field", "agg_field")),
+            DEFAULT_THRESHOLD_PARAM);
+    }
+
+    private void fieldScriptTestCase(Script script, long threshold) throws IOException {
+        final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        aggFieldType.setName("agg_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .script(script);
+
+        final int numDocs = randomIntBetween(100, 200);
+        int docsBelowThreshold = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            final long firstValue = randomLongBetween(0, 100);
+            final long secondValue = firstValue + 1;
+            if (firstValue < threshold && secondValue < threshold) {
+                docsBelowThreshold++;
+            }
+            docs.add(Set.of(
+                new SortedNumericDocValuesField(aggFieldType.name(), firstValue),
+                new SortedNumericDocValuesField(aggFieldType.name(), secondValue)
+            ));
+        }
+        final int finalDocsBelowThreshold = docsBelowThreshold;
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsBelowThreshold, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
+            },
+            singleton(aggFieldType)
+        );
+    }
+
+    private <B extends AggregationBuilder, I extends InternalAggregation> void testCase(
+            Query query,
+            B builder,
+            CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
+            Consumer<I> verify,
+            Collection<MappedFieldType> fieldTypes) throws IOException {
+
+        testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false);
+        testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true);
+    }
+
+    private <B extends AggregationBuilder, I extends InternalAggregation> void testCaseWithReduce(
+            Query query,
+            B builder,
+            CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
+            Consumer<I> verify,
+            Collection<MappedFieldType> fieldTypes,
+            boolean reduced) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                writeIndex.accept(indexWriter);
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                final IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+                final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]);
+                final I result;
                 if (reduced) {
-                    missing = searchAndReduce(indexSearcher, query, builder, fieldType);
+                    result = searchAndReduce(indexSearcher, query, builder, fieldTypesArray);
                 } else {
-                    missing = search(indexSearcher, query, builder, fieldType);
+                    result = search(indexSearcher, query, builder, fieldTypesArray);
                 }
-                verify.accept(missing);
+                verify.accept(result);
             }
         }
     }
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new MissingAggregationBuilder("_name", null)
+            .field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(
+            CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.BYTES,
+            CoreValuesSourceType.GEOPOINT,
+            CoreValuesSourceType.RANGE,
+            CoreValuesSourceType.HISTOGRAM
+        );
+    }
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        final Map<String, Function<Map<String, Object>, Object>> deterministicScripts = new HashMap<>();
+        deterministicScripts.put(VALUE_SCRIPT_PARAMS, vars -> {
+            final double value = ((Number) vars.get("_value")).doubleValue();
+            final long inc = ((Number) vars.get("inc")).longValue();
+            return value + inc;
+        });
+        deterministicScripts.put(VALUE_SCRIPT, vars -> {
+            final double value = ((Number) vars.get("_value")).doubleValue();
+            return value + DEFAULT_INC_PARAM;
+        });
+        deterministicScripts.put(FIELD_SCRIPT_PARAMS, vars -> {
+            final String fieldName = (String) vars.get("field");
+            final long threshold = ((Number) vars.get("threshold")).longValue();
+            return threshold(fieldName, threshold, vars);
+        });
+        deterministicScripts.put(FIELD_SCRIPT, vars -> {
+            final String fieldName = (String) vars.get("field");
+            return threshold(fieldName, DEFAULT_THRESHOLD_PARAM, vars);
+        });
+        final MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, deterministicScripts, emptyMap(), emptyMap());
+        final Map<String, ScriptEngine> engines = singletonMap(scriptEngine.getType(), scriptEngine);
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
+
+    private static List<Long> threshold(String fieldName, long threshold, Map<String, Object> vars) {
+        final LeafDocLookup lookup = (LeafDocLookup) vars.get("doc");
+        return lookup.get(fieldName).stream()
+            .map(value -> ((Number) value).longValue())
+            .filter(value -> value >= threshold)
+            .collect(toList());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index c67850436b393..04a0d71b650a9 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalAggregationsTests;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.suggest.SuggestTests;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -54,7 +55,8 @@ public QuerySearchResultTests() {
 
     private static QuerySearchResult createTestInstance() throws Exception {
         ShardId shardId = new ShardId("index", "uuid", randomInt());
-        QuerySearchResult result = new QuerySearchResult(randomLong(), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE));
+        QuerySearchResult result = new QuerySearchResult(new SearchContextId("", randomLong()),
+            new SearchShardTarget("node", shardId, null, OriginalIndices.NONE));
         if (randomBoolean()) {
             result.terminatedEarly(randomBoolean());
         }
@@ -75,7 +77,7 @@ public void testSerialization() throws Exception {
         QuerySearchResult querySearchResult = createTestInstance();
         Version version = VersionUtils.randomVersion(random());
         QuerySearchResult deserialized = copyWriteable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new, version);
-        assertEquals(querySearchResult.getRequestId(), deserialized.getRequestId());
+        assertEquals(querySearchResult.getContextId(), deserialized.getContextId());
         assertNull(deserialized.getSearchShardTarget());
         assertEquals(querySearchResult.topDocs().maxScore, deserialized.topDocs().maxScore, 0f);
         assertEquals(querySearchResult.topDocs().topDocs.totalHits, deserialized.topDocs().topDocs.totalHits);
diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 336a5500cfc71..404e8f92ec8f4 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -19,7 +19,12 @@
 
 package org.elasticsearch.search.query;
 
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.English;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
@@ -28,12 +33,17 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.NormalizingCharFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
@@ -42,11 +52,14 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WildcardQueryBuilder;
 import org.elasticsearch.index.query.WrapperQueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
@@ -58,16 +71,20 @@
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
+import java.io.Reader;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
 
+import static java.util.Collections.singletonMap;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -117,7 +134,7 @@ public class SearchQueryIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class);
+        return Arrays.asList(InternalSettingsPlugin.class, MockAnalysisPlugin.class);
     }
 
     @Override
@@ -1758,4 +1775,121 @@ public void testFieldAliasesForMetaFields() throws Exception {
        }
 
    }
+
+   /**
+    * Test that wildcard queries on keyword fields get normalized
+    */
+    public void testWildcardQueryNormalizationOnKeywordField() {
+       assertAcked(prepareCreate("test")
+               .setSettings(Settings.builder()
+                       .put("index.analysis.normalizer.lowercase_normalizer.type", "custom")
+                       .putList("index.analysis.normalizer.lowercase_normalizer.filter", "lowercase")
+                       .build())
+                .setMapping("field1", "type=keyword,normalizer=lowercase_normalizer"));
+       client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
+       refresh();
+
+        {
+            WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
+            SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+            assertHitCount(searchResponse, 1L);
+
+            wildCardQuery = wildcardQuery("field1", "bb*");
+            searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+            assertHitCount(searchResponse, 1L);
+        }
+   }
+
+    /**
+     * Test that wildcard queries on text fields get normalized
+     */
+     public void testWildcardQueryNormalizationOnTextField() {
+        assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                        .put("index.analysis.analyzer.lowercase_analyzer.type", "custom")
+                        .put("index.analysis.analyzer.lowercase_analyzer.tokenizer", "standard")
+                        .putList("index.analysis.analyzer.lowercase_analyzer.filter", "lowercase")
+                        .build())
+                 .setMapping("field1", "type=text,analyzer=lowercase_analyzer"));
+        client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
+        refresh();
+
+         {
+             WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
+             SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+             assertHitCount(searchResponse, 1L);
+
+             wildCardQuery = wildcardQuery("field1", "bb*");
+             searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+             assertHitCount(searchResponse, 1L);
+         }
+    }
+
+    /**
+     * Reserved characters should be excluded when the normalization is applied for keyword fields.
+     * See https://github.com/elastic/elasticsearch/issues/46300 for details.
+     */
+    public void testWildcardQueryNormalizationKeywordSpecialCharacters() {
+        assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder().put("index.analysis.char_filter.no_wildcard.type", "mock_pattern_replace")
+                        .put("index.analysis.normalizer.no_wildcard.type", "custom")
+                        .put("index.analysis.normalizer.no_wildcard.char_filter", "no_wildcard").build())
+                .setMapping("field", "type=keyword,normalizer=no_wildcard"));
+        client().prepareIndex("test").setId("1").setSource("field", "label-1").get();
+        refresh();
+
+        WildcardQueryBuilder wildCardQuery = wildcardQuery("field", "la*");
+        SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+        assertHitCount(searchResponse, 1L);
+
+        wildCardQuery = wildcardQuery("field", "la*el-?");
+        searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+        assertHitCount(searchResponse, 1L);
+    }
+
+    public static class MockAnalysisPlugin extends Plugin implements AnalysisPlugin {
+
+        @Override
+        public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
+            return singletonMap("mock_pattern_replace", (indexSettings, env, name, settings) -> {
+                class Factory implements NormalizingCharFilterFactory {
+
+                    private final Pattern pattern = Regex.compile("[\\*\\?]", null);
+
+                    @Override
+                    public String name() {
+                        return name;
+                    }
+
+                    @Override
+                    public Reader create(Reader reader) {
+                        return new PatternReplaceCharFilter(pattern, "", reader);
+                    }
+                }
+                return new Factory();
+            });
+        }
+
+        @Override
+        public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
+            return singletonMap("keyword", (indexSettings, environment, name, settings) -> TokenizerFactory.newFactory(name,
+                    () -> new MockTokenizer(MockTokenizer.KEYWORD, false)));
+        }
+    }
+
+    /**
+     * Test correct handling {@link SpanBooleanQueryRewriteWithMaxClause#rewrite(IndexReader, MultiTermQuery)}. That rewrite method is e.g.
+     * set for fuzzy queries with "constant_score" rewrite nested inside a `span_multi` query and would cause NPEs due to an unset
+     * {@link AttributeSource}.
+     */
+    public void testIssueFuzzyInsideSpanMulti() {
+        createIndex("test");
+        client().prepareIndex("test").setId("1").setSource("field", "foobarbaz").get();
+        ensureGreen();
+        refresh();
+
+        BoolQueryBuilder query = boolQuery().filter(spanMultiTermQueryBuilder(fuzzyQuery("field", "foobarbiz").rewrite("constant_score")));
+        SearchResponse response = client().prepareSearch("test").setQuery(query).get();
+        assertHitCount(response, 1);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index e11f74be0bab2..6e217a97ddf42 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -21,9 +21,11 @@
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -34,6 +36,7 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.rest.RestStatus;
@@ -41,6 +44,7 @@
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.junit.After;
 
@@ -655,6 +659,42 @@ public void testScrollRewrittenToMatchNoDocs() {
         }
     }
 
+    public void testRestartDataNodesDuringScrollSearch() throws Exception {
+        final String dataNode = internalCluster().startDataOnlyNode();
+        createIndex("demo", Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.routing.allocation.include._name", dataNode)
+            .build());
+        createIndex("prod", Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.routing.allocation.include._name", dataNode)
+            .build());
+        int numDocs = randomIntBetween(10, 100);
+        for (int i = 0; i < numDocs; i++) {
+            index("demo", "demo-" + i, Map.of());
+            index("prod", "prod-" + i, Map.of());
+        }
+        client().admin().indices().prepareRefresh().get();
+        SearchResponse respFromDemoIndex = client().prepareSearch("demo")
+            .setSize(randomIntBetween(1, 10))
+            .setQuery(new MatchAllQueryBuilder()).setScroll(TimeValue.timeValueMinutes(5)).get();
+
+        internalCluster().restartNode(dataNode, new InternalTestCluster.RestartCallback());
+        ensureGreen("demo", "prod");
+        SearchResponse respFromProdIndex = client().prepareSearch("prod")
+            .setSize(randomIntBetween(1, 10))
+            .setQuery(new MatchAllQueryBuilder()).setScroll(TimeValue.timeValueMinutes(5)).get();
+        assertNoFailures(respFromProdIndex);
+        SearchPhaseExecutionException error = expectThrows(SearchPhaseExecutionException.class,
+            () -> client().prepareSearchScroll(respFromDemoIndex.getScrollId()).get());
+        for (ShardSearchFailure shardSearchFailure : error.shardFailures()) {
+            assertThat(shardSearchFailure.getCause().getMessage(), containsString("No search context found for id [1]"));
+        }
+        client().prepareSearchScroll(respFromProdIndex.getScrollId()).get();
+    }
+
     private void assertToXContentResponse(ClearScrollResponse response, boolean succeed, int numFreed) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
index 44fe0d4dd5c58..ee74c37fbd401 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
@@ -66,7 +66,6 @@ public void testRestoreSnapshotRequestParsing() throws IOException {
         builder.field("rename_replacement", "rename-to");
         boolean partial = randomBoolean();
         builder.field("partial", partial);
-        builder.startObject("settings").field("set1", "val1").endObject();
         builder.startObject("index_settings").field("set1", "val2").endObject();
         if (randomBoolean()) {
             builder.field("ignore_index_settings", "set2,set3");
@@ -91,7 +90,6 @@ public void testRestoreSnapshotRequestParsing() throws IOException {
         assertEquals("rename-from", request.renamePattern());
         assertEquals("rename-to", request.renameReplacement());
         assertEquals(partial, request.partial());
-        assertEquals("val1", request.settings().get("set1"));
         assertArrayEquals(request.ignoreIndexSettings(), new String[]{"set2", "set3"});
         boolean expectedIgnoreAvailable = includeIgnoreUnavailable
             ? indicesOptions.ignoreUnavailable()
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index f8a4fe4b9429b..2e3102cb7e926 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -417,8 +417,8 @@ public void testRenderConnectionInfoXContent() throws IOException {
                 "\"num_nodes_connected\":2,\"max_connections_per_cluster\":3,\"initial_connect_timeout\":\"30m\"," +
                 "\"skip_unavailable\":true}}", Strings.toString(builder));
         } else {
-            assertEquals("{\"test_cluster\":{\"connected\":true,\"mode\":\"proxy\",\"address\":\"seed:1\"," +
-                "\"num_sockets_connected\":16,\"max_socket_connections\":18,\"initial_connect_timeout\":\"30m\"," +
+            assertEquals("{\"test_cluster\":{\"connected\":true,\"mode\":\"proxy\",\"proxy_address\":\"seed:1\"," +
+                "\"num_proxy_sockets_connected\":16,\"max_proxy_socket_connections\":18,\"initial_connect_timeout\":\"30m\"," +
                 "\"skip_unavailable\":true}}", Strings.toString(builder));
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 21078a085b017..75e577680274a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -728,6 +728,12 @@ public static String randomRealisticUnicodeOfCodepointLength(int codePoints) {
         return RandomizedTest.randomRealisticUnicodeOfCodepointLength(codePoints);
     }
 
+    /**
+     * @param maxArraySize The maximum number of elements in the random array
+     * @param stringSize The length of each String in the array
+     * @param allowNull Whether the returned array may be null
+     * @param allowEmpty Whether the returned array may be empty (have zero elements)
+     */
     public static String[] generateRandomStringArray(int maxArraySize, int stringSize, boolean allowNull, boolean allowEmpty) {
         if (allowNull && random().nextBoolean()) {
             return null;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index e7b5e937c0c02..12f06bd6a9deb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -52,6 +52,7 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -132,8 +133,8 @@ public Query buildFilteredQuery(Query query) {
     }
 
     @Override
-    public long id() {
-        return 0;
+    public SearchContextId id() {
+        return new SearchContextId("", 0);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
index 7e80810d7dd27..1449252d024ac 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
@@ -58,6 +58,13 @@ public static Map<String, Object> convertToMap(ToXContent part) throws IOExcepti
         return XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
     }
 
+    public static BytesReference convertToXContent(Map<String, ?> map, XContentType xContentType) throws IOException {
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
+            builder.map(map);
+            return BytesReference.bytes(builder);
+        }
+    }
+
 
     /**
      * Compares two maps generated from XContentObjects. The order of elements in arrays is ignored.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 2ecac6daac8f4..445fa98af651f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -1126,6 +1126,7 @@ protected static boolean isXPackTemplate(String name) {
         case ".logstash-management":
         case "security_audit_log":
         case ".slm-history":
+        case ".async-search":
             return true;
         default:
             return false;
diff --git a/x-pack/docs/en/security/authentication/custom-realm.asciidoc b/x-pack/docs/en/security/authentication/custom-realm.asciidoc
index 857444c0836d7..99f124b27397e 100644
--- a/x-pack/docs/en/security/authentication/custom-realm.asciidoc
+++ b/x-pack/docs/en/security/authentication/custom-realm.asciidoc
@@ -11,8 +11,7 @@ extension as part of an ordinary elasticsearch plugin.
 ==== Implementing a custom realm
 
 Sample code that illustrates the structure and implementation of a custom realm
-is provided in the https://github.com/elastic/shield-custom-realm-example[custom-realm-example]
-repository on GitHub. You can use this code as a starting point for creating your
+is provided in https://github.com/elastic/elasticsearch/tree/{branch}/x-pack/qa/security-example-spi-extension. You can use this code as a starting point for creating your
 own realm.
 
 To create a custom realm, you need to:
@@ -86,7 +85,7 @@ bin/elasticsearch-plugin install file:///<path>/my-realm-1.0.zip
 
 . Add a realm configuration of the appropriate realm type to `elasticsearch.yml`
 under the `xpack.security.authc.realms` namespace.
-You must define your realm within the namespace that matchesto the type defined
+You must define your realm within the namespace that matches the type defined
 by the extension.
 The options you can set depend on the settings exposed by the custom realm.
 At a minimum, you must explicitly set the `order` attribute to control the
diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
index c100ef161c755..1cf13fbd2a15d 100644
--- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc
+++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
@@ -879,7 +879,7 @@ an {es} access token and a refresh token. The access token is used as credential
 for subsequent calls to {es}. The refresh token enables the user to get new {es}
 access tokens after the current one expires.
 
-[[saml-no-kibana-realm]]]
+[[saml-no-kibana-realm]]
 ==== SAML realm
 
 You must create a SAML realm and configure it accordingly
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java
index 610cf89641188..eb7474b7bffd6 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetrics.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -20,7 +21,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -94,7 +94,7 @@ public Object getProperty(List<String> path) {
             return null;
         }
         assert topMetrics.size() == 1 : "property paths should only resolve against top metrics with size == 1.";
-        return topMetrics.get(0).metricValues[index];
+        return topMetrics.get(0).metricValues.get(index).numberValue();
     }
 
     @Override
@@ -161,13 +161,14 @@ public boolean equals(Object obj) {
     public double value(String name) {
         int index = metricNames.indexOf(name);
         if (index < 0) {
-            throw new IllegalArgumentException("unknown metric [" + name + "]");            
+            throw new IllegalArgumentException("unknown metric [" + name + "]");
         }
         if (topMetrics.isEmpty()) {
             return Double.NaN;
         }
         assert topMetrics.size() == 1 : "property paths should only resolve against top metrics with size == 1.";
-        return topMetrics.get(0).metricValues[index];
+        // TODO it'd probably be nicer to have "compareTo" instead of assuming a double.
+        return topMetrics.get(0).metricValues.get(index).numberValue().doubleValue();
     }
 
     SortOrder getSortOrder() {
@@ -206,9 +207,9 @@ TopMetric topMetric() {
     static class TopMetric implements Writeable, Comparable<TopMetric> {
         private final DocValueFormat sortFormat;
         private final SortValue sortValue;
-        private final double[] metricValues;
+        private final List<MetricValue> metricValues;
 
-        TopMetric(DocValueFormat sortFormat, SortValue sortValue, double[] metricValues) {
+        TopMetric(DocValueFormat sortFormat, SortValue sortValue, List<MetricValue> metricValues) {
             this.sortFormat = sortFormat;
             this.sortValue = sortValue;
             this.metricValues = metricValues;
@@ -217,14 +218,14 @@ static class TopMetric implements Writeable, Comparable<TopMetric> {
         TopMetric(StreamInput in) throws IOException {
             sortFormat = in.readNamedWriteable(DocValueFormat.class);
             sortValue = in.readNamedWriteable(SortValue.class);
-            metricValues = in.readDoubleArray();
+            metricValues = in.readList(s -> s.readOptionalWriteable(MetricValue::new));
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeNamedWriteable(sortFormat);
             out.writeNamedWriteable(sortValue);
-            out.writeDoubleArray(metricValues);
+            out.writeCollection(metricValues, StreamOutput::writeOptionalWriteable);
         }
 
         DocValueFormat getSortFormat() {
@@ -235,7 +236,7 @@ SortValue getSortValue() {
             return sortValue;
         }
 
-        double[] getMetricValues() {
+        List<MetricValue> getMetricValues() {
             return metricValues;
         }
 
@@ -246,9 +247,13 @@ public XContentBuilder toXContent(XContentBuilder builder, List<String> metricNa
                 sortValue.toXContent(builder, sortFormat);
                 builder.endArray();
                 builder.startObject(METRIC_FIELD.getPreferredName());
-                {
-                    for (int i = 0; i < metricValues.length; i++) {
-                        builder.field(metricNames.get(i), Double.isNaN(metricValues[i]) ? null : metricValues[i]);
+                for (int i = 0; i < metricValues.size(); i++) {
+                    MetricValue value = metricValues.get(i);
+                    builder.field(metricNames.get(i));
+                    if (value == null) {
+                        builder.nullValue();
+                    } else {
+                        value.toXContent(builder, ToXContent.EMPTY_PARAMS);
                     }
                 }
                 builder.endObject();
@@ -269,17 +274,79 @@ public boolean equals(Object obj) {
             TopMetric other = (TopMetric) obj;
             return sortFormat.equals(other.sortFormat)
                     && sortValue.equals(other.sortValue)
-                    && Arrays.equals(metricValues, other.metricValues);
+                    && metricValues.equals(other.metricValues);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(sortFormat, sortValue, Arrays.hashCode(metricValues));
+            return Objects.hash(sortFormat, sortValue, metricValues);
         }
 
         @Override
         public String toString() {
-            return "TopMetric[" + sortFormat + "," + sortValue + "," + Arrays.toString(metricValues) + "]"; 
+            return "TopMetric[" + sortFormat + "," + sortValue + "," + metricValues + "]"; 
+        }
+    }
+
+    static class MetricValue implements Writeable, ToXContent {
+        private final DocValueFormat format;
+        /**
+         * It is odd to have a "SortValue" be part of a MetricValue but it is
+         * a very convenient way to send a type-aware thing across the
+         * wire though. So here we are.
+         */
+        private final SortValue value;
+
+        MetricValue(DocValueFormat format, SortValue value) {
+            this.format = format;
+            this.value = value;
+        }
+
+        DocValueFormat getFormat() {
+            return format;
+        }
+
+        SortValue getValue() {
+            return value;
+        }
+
+        MetricValue(StreamInput in) throws IOException {
+            format = in.readNamedWriteable(DocValueFormat.class);
+            value = in.readNamedWriteable(SortValue.class);
+        }
+
+        Number numberValue() {
+            return value.numberValue();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeNamedWriteable(format);
+            out.writeNamedWriteable(value);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return value.toXContent(builder, format);
+        }
+
+        @Override
+        public String toString() {
+            return format + "," + value;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || obj.getClass() != getClass()) {
+                return false;
+            }
+            MetricValue other = (MetricValue) obj;
+            return format.equals(other.format) && value.equals(other.value);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(format, value);
         }
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java
index 300f583050837..e25a80b31e1bd 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregator.java
@@ -7,14 +7,16 @@
 package org.elasticsearch.xpack.analytics.topmetrics;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.BitArray;
 import org.elasticsearch.common.util.DoubleArray;
+import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -26,11 +28,17 @@
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.BucketedSort;
 import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.sort.SortValue;
+import org.elasticsearch.xpack.analytics.topmetrics.InternalTopMetrics.MetricValue;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.stream.Collectors.toList;
+
 /**
  * Collects the {@code top_metrics} aggregation, which functions like a memory
  * efficient but limited version of the {@code top_hits} aggregation. Amortized,
@@ -52,17 +60,16 @@ class TopMetricsAggregator extends NumericMetricsAggregator.MultiValue {
 
     TopMetricsAggregator(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData, int size,
-            SortBuilder<?> sort, List<String> metricNames, List<ValuesSource.Numeric> metricValuesSources) throws IOException {
+            SortBuilder<?> sort, List<MetricSource> metricSources) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         this.size = size;
-        assert metricNames.size() == metricValuesSources.size();
-        metrics = new Metrics(size, context.getQueryShardContext(), metricNames, metricValuesSources);
+        metrics = new Metrics(size, context.getQueryShardContext().bigArrays(), metricSources);
         /*
          * If we're only collecting a single value then only provided *that*
          * value to the sort so that swaps and loads are just a little faster
          * in that *very* common case.
          */
-        BucketedSort.ExtraData values = metricValuesSources.size() == 1 ? metrics.values[0] : metrics;
+        BucketedSort.ExtraData values = metricSources.size() == 1 ? metrics.values[0] : metrics;
         this.sort = sort.buildBucketedSort(context.getQueryShardContext(), size, values);
     }
 
@@ -71,7 +78,12 @@ public boolean hasMetric(String name) {
         if (size != 1) {
             throw new IllegalArgumentException("[top_metrics] can only the be target if [size] is [1] but was [" + size + "]");
         }
-        return metrics.names.contains(name);
+        for (MetricValues values : metrics.values) {
+            if (values.name().equals(name)) {
+                return true;
+            }
+        }
+        return false;
     }
 
     @Override
@@ -115,12 +127,12 @@ public void setScorer(Scorable s) throws IOException {
     public InternalAggregation buildAggregation(long bucket) throws IOException {
         List<InternalTopMetrics.TopMetric> topMetrics = sort.getValues(bucket, metrics.resultBuilder(sort.getFormat()));
         assert topMetrics.size() <= size;
-        return new InternalTopMetrics(name, sort.getOrder(), metrics.names, size, topMetrics, pipelineAggregators(), metaData());
+        return new InternalTopMetrics(name, sort.getOrder(), metrics.names(), size, topMetrics, pipelineAggregators(), metaData());
     }
 
     @Override
     public InternalTopMetrics buildEmptyAggregation() {
-        return InternalTopMetrics.buildEmptyAggregation(name, metrics.names, pipelineAggregators(), metaData());
+        return InternalTopMetrics.buildEmptyAggregation(name, metrics.names(), pipelineAggregators(), metaData());
     }
 
     @Override
@@ -128,21 +140,49 @@ public void doClose() {
         Releasables.close(sort, metrics);
     }
 
-    private static class Metrics implements BucketedSort.ExtraData, Releasable {
-        private final List<String> names;
+    /**
+     * Information about each metric that this {@link Aggregator} uses to
+     * load and format metric values.
+     */
+    static class MetricSource {
+        private final String name;
+        private final DocValueFormat format;
+        private final ValuesSource.Numeric valuesSource;
+
+        MetricSource(String name, DocValueFormat format, ValuesSource.Numeric valuesSource) {
+            this.name = name;
+            this.format = format;
+            this.valuesSource = valuesSource;
+        }
+
+        String getName() {
+            return name;
+        }
+
+        DocValueFormat getFormat() {
+            return format;
+        }
+    }
+
+    static class Metrics implements BucketedSort.ExtraData, Releasable {
         private final MetricValues[] values;
 
-        Metrics(int size, QueryShardContext ctx, List<String> names, List<ValuesSource.Numeric> valuesSources) {
-            this.names = names; 
-            values = new MetricValues[valuesSources.size()];
+        Metrics(int size, BigArrays bigArrays, List<MetricSource> sources) {
+            values = new MetricValues[sources.size()];
             int i = 0;
-            for (ValuesSource.Numeric valuesSource : valuesSources) {
-                if (valuesSource == null) {
-                    values[i++] = new MissingMetricValues();
-                    continue;
-                }
-                values[i++] = new CollectMetricValues(size, ctx.bigArrays(), valuesSource);
+            for (MetricSource source : sources) {
+                values[i++] = valuesFor(size, bigArrays, source);
+            }
+        }
+
+        private static MetricValues valuesFor(int size, BigArrays bigArrays, MetricSource source) {
+            if (source.valuesSource == null) {
+                return new AlwaysNullMetricValues(source);
+            }
+            if (source.valuesSource.isFloatingPoint()) {
+                return new DoubleMetricValues(size, bigArrays, source);
             }
+            return new LongMetricValues(size, bigArrays, source);
         }
 
         boolean needsScores() {
@@ -155,23 +195,28 @@ boolean needsScores() {
         }
 
         double metric(String name, long index) {
-            int valueIndex = names.indexOf(name);
-            if (valueIndex < 0) {
-                throw new IllegalArgumentException("[" + name + "] not found");
+            for (MetricValues value : values) {
+                if (value.name().equals(name)) {
+                    return value.doubleValue(index);
+                }
             }
-            return values[valueIndex].value(index);
+            throw new IllegalArgumentException("[" + name + "] not found");
         }
 
         BucketedSort.ResultBuilder<InternalTopMetrics.TopMetric> resultBuilder(DocValueFormat sortFormat) {
             return (index, sortValue) -> {
-                double[] result = new double[values.length];
+                List<InternalTopMetrics.MetricValue> result = new ArrayList<>(values.length);
                 for (int i = 0; i < values.length; i++) {
-                    result[i] = values[i].value(index);
+                    result.add(values[i].metricValue(index));
                 }
                 return new InternalTopMetrics.TopMetric(sortFormat, sortValue, result);
             };
         }
 
+        List<String> names() {
+            return Arrays.stream(values).map(MetricValues::name).collect(toList());
+        }
+
         @Override
         public void swap(long lhs, long rhs) {
             for (int i = 0; i < values.length; i++) {
@@ -198,32 +243,62 @@ public void close() {
         }
     }
 
-    private interface MetricValues extends BucketedSort.ExtraData, Releasable {
-        boolean needsScores();
-        double value(long index);
+    private abstract static class MetricValues implements BucketedSort.ExtraData, Releasable {
+        protected final MetricSource source;
+
+        MetricValues(MetricSource source) {
+            this.source = source;
+        }
+
+        final String name() {
+            return source.name;
+        }
+
+        abstract boolean needsScores();
+        abstract double doubleValue(long index);
+        abstract InternalTopMetrics.MetricValue metricValue(long index);
     }
-    private static class CollectMetricValues implements MetricValues {
-        private final BigArrays bigArrays;
-        private final ValuesSource.Numeric metricValueSource;
 
-        private DoubleArray values;
+    private abstract static class CollectingMetricValues extends MetricValues {
+        protected final BigArrays bigArrays;
 
-        CollectMetricValues(int size, BigArrays bigArrays, ValuesSource.Numeric metricValueSource) {
+        CollectingMetricValues(BigArrays bigArrays, MetricSource source) {
+            super(source);
             this.bigArrays = bigArrays;
-            this.metricValueSource = metricValueSource;
-            values = bigArrays.newDoubleArray(size, false);
         }
 
         @Override
-        public boolean needsScores() {
-            return metricValueSource.needsScores();
+        public final boolean needsScores() {
+            return source.valuesSource.needsScores();
+        }
+    }
+
+    /**
+     * Loads metrics for floating point numbers.
+     */
+    static class DoubleMetricValues extends CollectingMetricValues {
+        private DoubleArray values;
+
+        DoubleMetricValues(int size, BigArrays bigArrays, MetricSource source) {
+            super(bigArrays, source);
+            values = bigArrays.newDoubleArray(size, false);
         }
 
         @Override
-        public double value(long index) {
+        public double doubleValue(long index) {
             return values.get(index);
         }
 
+        @Override
+        public MetricValue metricValue(long index) {
+            double value = values.get(index);
+            if (Double.isNaN(value)) {
+                // Use NaN as a sentinel for "missing"
+                return null;
+            }
+            return new MetricValue(source.format, SortValue.from(value));
+        }
+
         @Override
         public void swap(long lhs, long rhs) {
             double tmp = values.get(lhs);
@@ -234,12 +309,13 @@ public void swap(long lhs, long rhs) {
         @Override
         public Loader loader(LeafReaderContext ctx) throws IOException {
             // TODO allow configuration of value mode
-            NumericDoubleValues metricValues = MultiValueMode.AVG.select(metricValueSource.doubleValues(ctx));
+            NumericDoubleValues metricValues = MultiValueMode.AVG.select(source.valuesSource.doubleValues(ctx));
             return (index, doc) -> {
                 if (index >= values.size()) {
                     values = bigArrays.grow(values, index + 1);
                 }
-                double metricValue = metricValues.advanceExact(doc) ? metricValues.doubleValue() : Double.NaN; 
+                // Use NaN as a sentinel for "missing"
+                double metricValue = metricValues.advanceExact(doc) ? metricValues.doubleValue() : Double.NaN;
                 values.set(index, metricValue);
             };
         }
@@ -249,12 +325,93 @@ public void close() {
             values.close();
         }
     }
-    private static class MissingMetricValues implements MetricValues {
+
+    /**
+     * Loads metrics for whole numbers.
+     */
+    static class LongMetricValues extends CollectingMetricValues {
+        /**
+         * Tracks "missing" values in a {@link BitArray}. Unlike
+         * {@link DoubleMetricValues}, we there isn't a sentinel value
+         * that we can steel from the longs to represent missing that
+         * won't lead to more trouble than it is worth. So we track
+         * "missing" values explicitly.
+         */
+        private final MissingHelper empty;
+        private LongArray values;
+
+        LongMetricValues(int size, BigArrays bigArrays, MetricSource source) {
+            super(bigArrays, source);
+            empty = new MissingHelper(bigArrays);
+            values = bigArrays.newLongArray(size, false);
+        }
+
+        @Override
+        public double doubleValue(long index) {
+            if (empty.isEmpty(index)) {
+                return Double.NaN;
+            }
+            return values.get(index);
+        }
+
+        @Override
+        public MetricValue metricValue(long index) {
+            if (empty.isEmpty(index)) {
+                return null;
+            }
+            return new MetricValue(source.format, SortValue.from(values.get(index)));
+        }
+
+        @Override
+        public void swap(long lhs, long rhs) {
+            long tmp = values.get(lhs);
+            values.set(lhs, values.get(rhs));
+            values.set(rhs, tmp);
+            empty.swap(lhs, rhs);
+        }
+
+        @Override
+        public Loader loader(LeafReaderContext ctx) throws IOException {
+            // TODO allow configuration of value mode
+            NumericDocValues metricValues = MultiValueMode.AVG.select(source.valuesSource.longValues(ctx));
+            return (index, doc) -> {
+                if (false == metricValues.advanceExact(doc)) {
+                    empty.markMissing(index);
+                    return;
+                }
+                if (index >= values.size()) {
+                    values = bigArrays.grow(values, index + 1);
+                }
+                values.set(index, metricValues.longValue());
+                empty.markNotMissing(index);
+            };
+        }
+
+        @Override
+        public void close() {
+            Releasables.close(values, empty);
+        }
+    }
+
+    /**
+     * {@linkplain MetricValues} implementation for unmapped fields
+     * that always returns {@code null} or {@code NaN}.
+     */
+    static class AlwaysNullMetricValues extends MetricValues {
+        AlwaysNullMetricValues(MetricSource source) {
+            super(source);
+        }
+
         @Override
-        public double value(long index) {
+        public double doubleValue(long index) {
             return Double.NaN;
         }
 
+        @Override
+        public MetricValue metricValue(long index) {
+            return null;
+        }
+
         @Override
         public boolean needsScores() {
             return false;
@@ -268,8 +425,76 @@ public Loader loader(LeafReaderContext ctx) throws IOException {
             return (index, doc) -> {};
         }
 
+        @Override
+        public void close() {}
+    }
+
+    /**
+     * Helps {@link LongMetricValues} track "empty" slots. It attempts to have
+     * very low CPU overhead and no memory overhead when there *aren't* empty
+     * values.
+     */
+    private static class MissingHelper implements Releasable {
+        private final BigArrays bigArrays;
+        private BitArray tracker;
+
+        MissingHelper(BigArrays bigArrays) {
+            this.bigArrays = bigArrays;
+        }
+
+        void markMissing(long index) {
+            int i = asInt(index);
+            if (tracker == null) {
+                tracker = new BitArray(i, bigArrays);
+            }
+            tracker.set(i);
+        }
+
+        void markNotMissing(long index) {
+            if (tracker == null) {
+                return;
+            }
+            tracker.clear(asInt(index));
+        }
+
+        void swap(long lhs, long rhs) {
+            if (tracker == null) {
+                return;
+            }
+            int l = asInt(lhs);
+            int r = asInt(rhs);
+            boolean backup = tracker.get(l);
+            if (tracker.get(r)) {
+                tracker.set(l);
+            } else {
+                tracker.clear(l);
+            }
+            if (backup) {
+                tracker.set(r);
+            } else {
+                tracker.clear(r);
+            }
+        }
+
+        boolean isEmpty(long index) {
+            if (tracker == null) {
+                return false;
+            }
+            return tracker.get(asInt(index));
+        }
+
         @Override
         public void close() {
+            if (tracker != null) {
+                tracker.close();
+            }
+        }
+
+        private int asInt(long index) {
+            if (index > Integer.MAX_VALUE) {
+                throw new IllegalArgumentException("top_metrics can't track more than " + Integer.MAX_VALUE + " values.");
+            }
+            return (int) index;
         }
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java
index be63f886dede5..2c2599e450a97 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java
@@ -57,14 +57,14 @@ protected TopMetricsAggregator createInternal(SearchContext searchContext, Aggre
                     + "]. This limit can be set by changing the [" + MAX_BUCKET_SIZE.getKey()
                     + "] index level setting.");
         }
-        List<String> metricNames = metricFields.stream().map(MultiValuesSourceFieldConfig::getFieldName).collect(toList());
-        List<ValuesSource.Numeric> metricValuesSources = metricFields.stream().map(config -> {
+        List<TopMetricsAggregator.MetricSource> metricSources = metricFields.stream().map(config -> {
                     ValuesSourceConfig<ValuesSource.Numeric> resolved = ValuesSourceConfig.resolve(
                             searchContext.getQueryShardContext(), ValueType.NUMERIC,
                             config.getFieldName(), config.getScript(), config.getMissing(), config.getTimeZone(), null);
-                    return resolved.toValuesSource(searchContext.getQueryShardContext());
+                    return new TopMetricsAggregator.MetricSource(config.getFieldName(), resolved.format(),
+                            resolved.toValuesSource(searchContext.getQueryShardContext()));
                 }).collect(toList());
         return new TopMetricsAggregator(name, searchContext, parent, pipelineAggregators, metaData, size,
-                sortBuilders.get(0), metricNames, metricValuesSources);
+                sortBuilders.get(0), metricSources);
     }
 }
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java
index 668b6a3aea8e8..eae410e015dfa 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsReduceTests.java
@@ -96,7 +96,9 @@ private InternalTopMetrics buildFilled(SortOrder sortOrder, int size, InternalTo
 
     private InternalTopMetrics.TopMetric top(SortValue sortValue, double metricValue) {
         DocValueFormat sortFormat = randomFrom(DocValueFormat.RAW, DocValueFormat.BINARY, DocValueFormat.BOOLEAN, DocValueFormat.IP);
-        return new InternalTopMetrics.TopMetric(sortFormat, sortValue, new double[] {metricValue});
+        DocValueFormat metricFormat = randomFrom(DocValueFormat.RAW, DocValueFormat.BINARY, DocValueFormat.BOOLEAN, DocValueFormat.IP);
+        InternalTopMetrics.MetricValue realMetricValue = new InternalTopMetrics.MetricValue(metricFormat, SortValue.from(metricValue));
+        return new InternalTopMetrics.TopMetric(sortFormat, sortValue, singletonList(realMetricValue));
     }
 
     private InternalTopMetrics reduce(InternalTopMetrics... results) {
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
index c8bda2dabec0e..64c041604fc5d 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
@@ -31,13 +31,15 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.IntStream;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 
 public class InternalTopMetricsTests extends InternalAggregationTestCase<InternalTopMetrics> {
@@ -48,7 +50,11 @@ public class InternalTopMetricsTests extends InternalAggregationTestCase<Interna
      * would fail because the instances that it attempts to reduce don't
      * have their results in the same order.
      */
-    private SortOrder sortOrder = randomFrom(SortOrder.values());
+    private final SortOrder sortOrder = randomFrom(SortOrder.values());
+    private final InternalTopMetrics.MetricValue metricOneDouble =
+            new InternalTopMetrics.MetricValue(DocValueFormat.RAW, SortValue.from(1.0));
+    private final InternalTopMetrics.MetricValue metricOneLong =
+            new InternalTopMetrics.MetricValue(DocValueFormat.RAW, SortValue.from(1));
 
     public void testEmptyIsNotMapped() {
         InternalTopMetrics empty = InternalTopMetrics.buildEmptyAggregation(
@@ -62,8 +68,9 @@ public void testNonEmptyIsMapped() {
     }
 
     public void testToXContentDoubleSortValue() throws IOException {
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1,
-                List.of(new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {1.0})), emptyList(), null);
+        List<InternalTopMetrics.TopMetric> top = singletonList(
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricOneDouble)));
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -81,12 +88,11 @@ public void testToXContentDoubleSortValue() throws IOException {
                 "}"));
     }
 
-    public void testToXConentDateSortValue() throws IOException {
-        DocValueFormat sortFormat = new DocValueFormat.DateTime(DateFormatter.forPattern("strict_date_time"), ZoneId.of("UTC"),
-                DateFieldMapper.Resolution.MILLISECONDS);
+    public void testToXContentDateSortValue() throws IOException {
         SortValue sortValue = SortValue.from(ZonedDateTime.parse("2007-12-03T10:15:30Z").toInstant().toEpochMilli());
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1,
-                List.of(new InternalTopMetrics.TopMetric(sortFormat, sortValue, new double[] {1.0})), emptyList(), null);
+        List<InternalTopMetrics.TopMetric> top = singletonList(new InternalTopMetrics.TopMetric(
+                strictDateTime(), sortValue, singletonList(metricOneDouble)));
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -104,10 +110,54 @@ public void testToXConentDateSortValue() throws IOException {
                 "}"));
     }
 
+    public void testToXContentLongMetricValue() throws IOException {
+        List<InternalTopMetrics.TopMetric> top = singletonList(
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricOneLong)));
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
+        assertThat(Strings.toString(tm, true, true), equalTo(
+                "{\n" +
+                "  \"test\" : {\n" +
+                "    \"top\" : [\n" +
+                "      {\n" +
+                "        \"sort\" : [\n" +
+                "          1.0\n" +
+                "        ],\n" +
+                "        \"metrics\" : {\n" +
+                "          \"test\" : 1\n" +
+                "        }\n" +
+                "      }\n" +
+                "    ]\n" +
+                "  }\n" +
+                "}"));
+    }
+
+    public void testToXContentDateMetricValue() throws IOException {
+        InternalTopMetrics.MetricValue metricValue = new InternalTopMetrics.MetricValue(
+                strictDateTime(), SortValue.from(ZonedDateTime.parse("2007-12-03T10:15:30Z").toInstant().toEpochMilli()));
+        List<InternalTopMetrics.TopMetric> top = singletonList(
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricValue)));
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 1, top, emptyList(), null);
+        assertThat(Strings.toString(tm, true, true), equalTo(
+                "{\n" +
+                "  \"test\" : {\n" +
+                "    \"top\" : [\n" +
+                "      {\n" +
+                "        \"sort\" : [\n" +
+                "          1.0\n" +
+                "        ],\n" +
+                "        \"metrics\" : {\n" +
+                "          \"test\" : \"2007-12-03T10:15:30.000Z\"\n" +
+                "        }\n" +
+                "      }\n" +
+                "    ]\n" +
+                "  }\n" +
+                "}"));
+    }
+
     public void testToXContentManyMetrics() throws IOException {
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, List.of("foo", "bar", "baz"), 1,
-                List.of(new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {1.0, 2.0, 3.0})),
-                emptyList(), null);
+        List<InternalTopMetrics.TopMetric> top = singletonList(new InternalTopMetrics.TopMetric(
+                DocValueFormat.RAW, SortValue.from(1.0), List.of(metricOneDouble, metricOneLong, metricOneDouble)));
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, List.of("foo", "bar", "baz"), 1, top, emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -118,8 +168,8 @@ public void testToXContentManyMetrics() throws IOException {
                 "        ],\n" +
                 "        \"metrics\" : {\n" +
                 "          \"foo\" : 1.0,\n" +
-                "          \"bar\" : 2.0,\n" +
-                "          \"baz\" : 3.0\n" +
+                "          \"bar\" : 1,\n" +
+                "          \"baz\" : 1.0\n" +
                 "        }\n" +
                 "      }\n" +
                 "    ]\n" +
@@ -128,11 +178,10 @@ public void testToXContentManyMetrics() throws IOException {
     }
 
     public void testToXContentManyTopMetrics() throws IOException {
-        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 2,
-                List.of(
-                    new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {1.0}),
-                    new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(2.0), new double[] {2.0})),
-                emptyList(), null);
+        List<InternalTopMetrics.TopMetric> top = List.of(
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), singletonList(metricOneDouble)),
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(2.0), singletonList(metricOneLong)));
+        InternalTopMetrics tm = new InternalTopMetrics("test", sortOrder, singletonList("test"), 2, top, emptyList(), null);
         assertThat(Strings.toString(tm, true, true), equalTo(
                 "{\n" +
                 "  \"test\" : {\n" +
@@ -150,7 +199,7 @@ public void testToXContentManyTopMetrics() throws IOException {
                 "          2.0\n" +
                 "        ],\n" +
                 "        \"metrics\" : {\n" +
-                "          \"test\" : 2.0\n" +
+                "          \"test\" : 1\n" +
                 "        }\n" +
                 "      }\n" +
                 "    ]\n" +
@@ -169,10 +218,15 @@ protected List<NamedXContentRegistry.Entry> getNamedXContents() {
     @Override
     protected InternalTopMetrics createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
+        return createTestInstance(name, pipelineAggregators, metaData, InternalAggregationTestCase::randomNumericDocValueFormat);
+    }
+
+    private InternalTopMetrics createTestInstance(String name, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData, Supplier<DocValueFormat> randomDocValueFormat) {
         int metricCount = between(1, 5);
         List<String> metricNames = randomMetricNames(metricCount);
         int size = between(1, 100);
-        List<InternalTopMetrics.TopMetric> topMetrics = randomTopMetrics(between(0, size), metricCount);
+        List<InternalTopMetrics.TopMetric> topMetrics = randomTopMetrics(randomDocValueFormat, between(0, size), metricCount);
         return new InternalTopMetrics(name, sortOrder, metricNames, size, topMetrics, pipelineAggregators, metaData);
     }
 
@@ -201,7 +255,8 @@ protected InternalTopMetrics mutateInstance(InternalTopMetrics instance) throws
         case 4:
             int fixedSize = size;
             int fixedMetricsSize = metricNames.size();
-            topMetrics = randomValueOtherThan(topMetrics, () -> randomTopMetrics(between(1, fixedSize), fixedMetricsSize));
+            topMetrics = randomValueOtherThan(topMetrics, () -> randomTopMetrics(
+                    InternalAggregationTestCase::randomNumericDocValueFormat, between(1, fixedSize), fixedMetricsSize));
             break;
         default:
             throw new IllegalArgumentException("bad mutation");
@@ -215,6 +270,18 @@ protected Reader<InternalTopMetrics> instanceReader() {
         return InternalTopMetrics::new;
     }
 
+    /**
+     * An extra test for parsing dates from xcontent because we can't random
+     * into {@link DocValueFormat.DateTime} because it doesn't
+     * implement {@link Object#equals(Object)}.
+     */
+    public void testFromXContentDates() throws IOException {
+        InternalTopMetrics aggregation = createTestInstance(
+                randomAlphaOfLength(3), emptyList(), emptyMap(), InternalTopMetricsTests::strictDateTime);
+        ParsedAggregation parsedAggregation = parseAndAssert(aggregation, randomBoolean(), randomBoolean());
+        assertFromXContent(aggregation, parsedAggregation);
+    }
+
     @Override
     protected void assertFromXContent(InternalTopMetrics aggregation, ParsedAggregation parsedAggregation) throws IOException {
         ParsedTopMetrics parsed = (ParsedTopMetrics) parsedAggregation;
@@ -228,8 +295,14 @@ protected void assertFromXContent(InternalTopMetrics aggregation, ParsedAggregat
             assertThat(parsedTop.getSort(), equalTo(singletonList(expectedSort)));
             assertThat(parsedTop.getMetrics().keySet(), hasSize(aggregation.getMetricNames().size()));
             for (int m = 0; m < aggregation.getMetricNames().size(); m++) {
-                assertThat(parsedTop.getMetrics(),
-                        hasEntry(aggregation.getMetricNames().get(m), internalTop.getMetricValues()[m]));
+                String name = aggregation.getMetricNames().get(m);
+                InternalTopMetrics.MetricValue value = internalTop.getMetricValues().get(m);
+                assertThat(parsedTop.getMetrics(), hasKey(name));
+                if (value.getFormat() == DocValueFormat.RAW) {
+                    assertThat(parsedTop.getMetrics().get(name), equalTo(value.numberValue()));
+                } else {
+                    assertThat(parsedTop.getMetrics().get(name), equalTo(value.getValue().format(value.getFormat())));
+                }
             }
         }
     }
@@ -249,10 +322,11 @@ protected void assertReduced(InternalTopMetrics reduced, List<InternalTopMetrics
         assertThat(reduced.getTopMetrics(), equalTo(winners));
     }
 
-    private List<InternalTopMetrics.TopMetric> randomTopMetrics(int length, int metricCount) {
+    private List<InternalTopMetrics.TopMetric> randomTopMetrics(
+            Supplier<DocValueFormat> randomDocValueFormat, int length, int metricCount) {
         return IntStream.range(0, length)
                 .mapToObj(i -> new InternalTopMetrics.TopMetric(
-                        randomNumericDocValueFormat(), randomSortValue(), randomMetricValues(metricCount)
+                        randomDocValueFormat.get(), randomSortValue(), randomMetricValues(randomDocValueFormat, metricCount)
                 ))
                 .sorted((lhs, rhs) -> sortOrder.reverseMul() * lhs.getSortValue().compareTo(rhs.getSortValue()))
                 .collect(toList());
@@ -266,10 +340,18 @@ static List<String> randomMetricNames(int metricCount) {
         return new ArrayList<>(names);
     }
 
-    private double[] randomMetricValues(int metricCount) {
-        return IntStream.range(0, metricCount).mapToDouble(i -> randomDouble()).toArray();
+    private List<InternalTopMetrics.MetricValue> randomMetricValues(Supplier<DocValueFormat> randomDocValueFormat, int metricCount) {
+        return IntStream.range(0, metricCount)
+                .mapToObj(i -> new InternalTopMetrics.MetricValue(randomDocValueFormat.get(), randomSortValue()))
+                .collect(toList());
     }
 
+    private static DocValueFormat strictDateTime() {
+        return new DocValueFormat.DateTime(
+            DateFormatter.forPattern("strict_date_time"), ZoneId.of("UTC"), DateFieldMapper.Resolution.MILLISECONDS);
+    }
+
+
     private static SortValue randomSortValue() {
         if (randomBoolean()) {
             return SortValue.from(randomLong());
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java
new file mode 100644
index 0000000000000..f9e2e34140f5d
--- /dev/null
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorMetricsTests.java
@@ -0,0 +1,213 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.analytics.topmetrics;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.sort.SortValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.analytics.topmetrics.InternalTopMetrics.MetricValue;
+import org.elasticsearch.xpack.analytics.topmetrics.InternalTopMetrics.TopMetric;
+import org.elasticsearch.xpack.analytics.topmetrics.TopMetricsAggregator.MetricSource;
+import org.elasticsearch.xpack.analytics.topmetrics.TopMetricsAggregator.Metrics;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.IntStream;
+
+import static java.util.Collections.singletonList;
+import static java.util.stream.Collectors.toList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notANumber;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TopMetricsAggregatorMetricsTests extends ESTestCase {
+    public void testUnmapped() throws IOException {
+        withMetric(null, (m, source) -> {
+            // Load from doc is a noop
+            m.loader(null).loadFromDoc(0, randomInt());
+            assertNullMetric(m, source, randomInt());
+        });
+    }
+
+    public void testEmptyLong() throws IOException {
+        SortedNumericDocValues values = mock(SortedNumericDocValues.class);
+        when(values.advanceExact(0)).thenReturn(false);
+        withMetric(valuesSource(values), (m, source) -> {
+            m.loader(null).loadFromDoc(0, 0);
+            assertNullMetric(m, source, 0);
+        });
+    }
+
+    public void testEmptyDouble() throws IOException {
+        SortedNumericDoubleValues values = mock(SortedNumericDoubleValues.class);
+        when(values.advanceExact(0)).thenReturn(false);
+        withMetric(valuesSource(values), (m, source) -> {
+            m.loader(null).loadFromDoc(0, 0);
+            assertNullMetric(m, source, 0);
+        });
+    }
+
+    public void testLoadLong() throws IOException {
+        long value = randomLong();
+        SortedNumericDocValues values = mock(SortedNumericDocValues.class);
+        when(values.advanceExact(0)).thenReturn(true);
+        when(values.docValueCount()).thenReturn(1);
+        when(values.nextValue()).thenReturn(value);
+        withMetric(valuesSource(values), (m, source) -> {
+            m.loader(null).loadFromDoc(0, 0);
+            assertMetricValue(m, 0, source, SortValue.from(value));
+        });
+    }
+
+    public void testLoadDouble() throws IOException {
+        double value = randomDouble();
+        SortedNumericDoubleValues values = mock(SortedNumericDoubleValues.class);
+        when(values.advanceExact(0)).thenReturn(true);
+        when(values.docValueCount()).thenReturn(1);
+        when(values.nextValue()).thenReturn(value);
+        withMetric(valuesSource(values), (m, source) -> {
+            m.loader(null).loadFromDoc(0, 0);
+            assertMetricValue(m, 0, source, SortValue.from(value));
+        });
+    }
+
+    public void testLoadAndSwapLong() throws IOException {
+        long firstValue = randomLong();
+        long secondValue = randomLong();
+        SortedNumericDocValues values = mock(SortedNumericDocValues.class);
+        when(values.advanceExact(0)).thenReturn(true);
+        when(values.advanceExact(1)).thenReturn(true);
+        when(values.docValueCount()).thenReturn(1);
+        when(values.nextValue()).thenReturn(firstValue, secondValue);
+        withMetric(valuesSource(values), (m, source) -> {
+            assertLoadTwoAndSwap(m, source, SortValue.from(firstValue), SortValue.from(secondValue));
+        });
+    }
+
+    public void testLoadAndSwapDouble() throws IOException {
+        double firstValue = randomDouble();
+        double secondValue = randomDouble();
+        SortedNumericDoubleValues values = mock(SortedNumericDoubleValues.class);
+        when(values.advanceExact(0)).thenReturn(true);
+        when(values.advanceExact(1)).thenReturn(true);
+        when(values.docValueCount()).thenReturn(1);
+        when(values.nextValue()).thenReturn(firstValue, secondValue);
+        withMetric(valuesSource(values), (m, source) -> {
+            assertLoadTwoAndSwap(m, source, SortValue.from(firstValue), SortValue.from(secondValue));
+        });
+    }
+
+    public void testManyValues() throws IOException {
+        long[] values = IntStream.range(0, between(2, 100)).mapToLong(i -> randomLong()).toArray();
+        List<ValuesSource.Numeric> valuesSources = Arrays.stream(values)
+                .mapToObj(v -> {
+                    try {
+                        SortedNumericDocValues docValues = mock(SortedNumericDocValues.class);
+                        when(docValues.advanceExact(0)).thenReturn(true);
+                        when(docValues.docValueCount()).thenReturn(1);
+                        when(docValues.nextValue()).thenReturn(v);
+                        return valuesSource(docValues);
+                    } catch (IOException e) {
+                        throw new AssertionError(e);
+                    }
+                })
+                .collect(toList());
+        withMetrics(valuesSources, (m, sources) -> {
+            m.loader(null).loadFromDoc(0, 0);
+            TopMetric metric = m.resultBuilder(DocValueFormat.RAW).build(0, SortValue.from(1));
+            assertThat(metric.getMetricValues(), hasSize(values.length));
+            for (int i = 0; i < values.length; i++) {
+                MetricSource source = sources.get(i);
+                assertThat(m.metric(source.getName(), 0), equalTo((double) values[i]));
+                assertThat(metric.getMetricValues(),
+                        hasItem(new MetricValue(source.getFormat(), SortValue.from(values[i]))));
+            }
+        });
+    }
+
+    private ValuesSource.Numeric valuesSource(SortedNumericDocValues values) throws IOException {
+        ValuesSource.Numeric source = mock(ValuesSource.Numeric.class);
+        when(source.isFloatingPoint()).thenReturn(false);
+        when(source.longValues(null)).thenReturn(values);
+        return source;
+    }
+
+    private ValuesSource.Numeric valuesSource(SortedNumericDoubleValues values) throws IOException {
+        ValuesSource.Numeric source = mock(ValuesSource.Numeric.class);
+        when(source.isFloatingPoint()).thenReturn(true);
+        when(source.doubleValues(null)).thenReturn(values);
+        return source;
+    }
+
+    private void withMetric(ValuesSource.Numeric valuesSource,
+            CheckedBiConsumer<Metrics, MetricSource, IOException> consumer) throws IOException {
+        withMetrics(singletonList(valuesSource), (m, sources) -> consumer.accept(m, sources.get(0)));
+    }
+
+    private void withMetrics(List<ValuesSource.Numeric> valuesSources,
+            CheckedBiConsumer<Metrics, List<MetricSource>, IOException> consumer) throws IOException {
+        Set<String> names = new HashSet<>();
+        List<MetricSource> sources = new ArrayList<>(valuesSources.size());
+        for (ValuesSource.Numeric valuesSource : valuesSources) {
+            String name = randomValueOtherThanMany(names::contains, () -> randomAlphaOfLength(5));
+            names.add(name);
+            sources.add(new MetricSource(name, randomDocValueFormat(), valuesSource));
+        }
+        try (Metrics m = new Metrics(1, BigArrays.NON_RECYCLING_INSTANCE, sources)) {
+            consumer.accept(m, sources);
+        }
+    }
+
+    private void assertNullMetric(Metrics m, MetricSource source, long index) {
+        DocValueFormat sortFormat = randomDocValueFormat();
+        assertThat(m.metric(source.getName(), index), notANumber());
+        TopMetric metric = m.resultBuilder(sortFormat).build(index, SortValue.from(1));
+        assertThat(metric.getSortFormat(), sameInstance(sortFormat));
+        assertThat(metric.getMetricValues(), equalTo(singletonList(null)));
+    }
+
+    private void assertMetricValue(Metrics m, long index, MetricSource source, SortValue value) {
+        DocValueFormat sortFormat = randomDocValueFormat();
+        assertThat(m.metric(source.getName(), index), equalTo(value.numberValue().doubleValue()));
+        TopMetric metric = m.resultBuilder(sortFormat).build(index, SortValue.from(1));
+        assertThat(metric.getSortValue(), equalTo(SortValue.from(1)));
+        assertThat(metric.getSortFormat(), sameInstance(sortFormat));
+        assertThat(metric.getMetricValues(), equalTo(singletonList(new MetricValue(source.getFormat(), value))));
+    }
+
+    private void assertLoadTwoAndSwap(Metrics m, MetricSource source, SortValue firstValue, SortValue secondValue) throws IOException {
+        m.loader(null).loadFromDoc(0, 0);
+        m.loader(null).loadFromDoc(1, 1);
+        assertMetricValue(m, 0, source, firstValue);
+        assertMetricValue(m, 1, source, secondValue);
+        m.swap(0, 1);
+        assertMetricValue(m, 0, source, secondValue);
+        assertMetricValue(m, 1, source, firstValue);
+        m.loader(null).loadFromDoc(2, 2); // 2 is empty
+        assertNullMetric(m, source, 2);
+        m.swap(0, 2);
+        assertNullMetric(m, source, 0);
+        assertMetricValue(m, 2, source, secondValue);
+    }
+
+    private DocValueFormat randomDocValueFormat() {
+        return randomFrom(DocValueFormat.RAW, DocValueFormat.BINARY, DocValueFormat.BOOLEAN);
+    }
+}
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
index 75f91adf569f9..39dc9bf5e9fda 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
@@ -78,7 +78,6 @@
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.notANumber;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -97,10 +96,12 @@ public void testUnmappedMetric() throws IOException {
                 },
                 numberFieldType(NumberType.DOUBLE, "s"));
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
-        assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.0, Double.NaN))));
+        assertThat(result.getTopMetrics(), hasSize(1));
+        assertThat(result.getTopMetrics().get(0).getSortValue(), equalTo(SortValue.from(1.0)));
+        assertThat(result.getTopMetrics().get(0).getMetricValues(), equalTo(singletonList(null)));
     }
 
-    public void testMissingValueForMetric() throws IOException {
+    public void testMissingValueForDoubleMetric() throws IOException {
         InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(singletonList(doubleField("s", 1.0)));
                 },
@@ -108,11 +109,21 @@ public void testMissingValueForMetric() throws IOException {
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
         assertThat(result.getTopMetrics(), hasSize(1));
         assertThat(result.getTopMetrics().get(0).getSortValue(), equalTo(SortValue.from(1.0)));
-        assertThat(result.getTopMetrics().get(0).getMetricValues().length, equalTo(1));
-        assertThat(result.getTopMetrics().get(0).getMetricValues()[0], notANumber());
+        assertThat(result.getTopMetrics().get(0).getMetricValues(), equalTo(singletonList(null)));
     }
 
-    public void testActualValueForMetric() throws IOException {
+    public void testMissingValueForLongMetric() throws IOException {
+        InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
+                    writer.addDocument(singletonList(longField("s", 1)));
+                },
+                longFields());
+        assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
+        assertThat(result.getTopMetrics(), hasSize(1));
+        assertThat(result.getTopMetrics().get(0).getSortValue(), equalTo(SortValue.from(1)));
+        assertThat(result.getTopMetrics().get(0).getMetricValues(), equalTo(singletonList(null)));
+    }
+
+    public void testActualValueForDoubleMetric() throws IOException {
         InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(Arrays.asList(doubleField("s", 1.0), doubleField("m", 2.0)));
                 },
@@ -121,6 +132,15 @@ public void testActualValueForMetric() throws IOException {
         assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.0, 2.0))));
     }
 
+    public void testActualValueForLongMetric() throws IOException {
+        InternalTopMetrics result = collect(simpleBuilder(), new MatchAllDocsQuery(), writer -> {
+                    writer.addDocument(Arrays.asList(longField("s", 1), longField("m", 2)));
+                },
+                longFields());
+        assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
+        assertThat(result.getTopMetrics(), equalTo(singletonList(top(1, 2))));
+    }
+
     private InternalTopMetrics collectFromDoubles(TopMetricsAggregationBuilder builder) throws IOException {
         return collect(builder, new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(Arrays.asList(doubleField("s", 1.0), doubleField("m", 2.0)));
@@ -132,8 +152,7 @@ private InternalTopMetrics collectFromDoubles(TopMetricsAggregationBuilder build
     public void testSortByDoubleAscending() throws IOException {
         InternalTopMetrics result = collectFromDoubles(simpleBuilder(new FieldSortBuilder("s").order(SortOrder.ASC)));
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
-        assertThat(result.getTopMetrics(), equalTo(singletonList(
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {2.0}))));
+        assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.0, 2.0))));
     }
 
     public void testSortByDoubleDescending() throws IOException {
@@ -387,13 +406,14 @@ public void testManyMetrics() throws IOException {
                 ));
         InternalTopMetrics result = collect(builder, new MatchAllDocsQuery(), writer -> {
                     writer.addDocument(Arrays.asList(doubleField("s", 1.0),
-                            doubleField("m1", 12.0), doubleField("m2", 22.0), doubleField("m3", 32.0)));
+                            doubleField("m1", 12.0), longField("m2", 22), doubleField("m3", 32.0)));
                     writer.addDocument(Arrays.asList(doubleField("s", 2.0),
-                            doubleField("m1", 13.0), doubleField("m2", 23.0), doubleField("m3", 33.0)));
+                            doubleField("m1", 13.0), longField("m2", 23), doubleField("m3", 33.0)));
                 }, manyMetricsFields());
         assertThat(result.getSortOrder(), equalTo(SortOrder.ASC));
         assertThat(result.getTopMetrics(), equalTo(singletonList(
-                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), new double[] {12.0, 22.0, 32.0}))));
+                new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(1.0), metricValues(
+                        SortValue.from(12.0), SortValue.from(22), SortValue.from(32.0))))));
     }
 
     private TopMetricsAggregationBuilder simpleBuilder() {
@@ -426,11 +446,15 @@ private MappedFieldType[] doubleFields() {
         return new MappedFieldType[] {numberFieldType(NumberType.DOUBLE, "s"), numberFieldType(NumberType.DOUBLE, "m")};
     }
 
+    private MappedFieldType[] longFields() {
+        return new MappedFieldType[] {numberFieldType(NumberType.LONG, "s"), numberFieldType(NumberType.LONG, "m")};
+    }
+
     private MappedFieldType[] manyMetricsFields() {
         return new MappedFieldType[] {
                 numberFieldType(NumberType.DOUBLE, "s"),
                 numberFieldType(NumberType.DOUBLE, "m1"),
-                numberFieldType(NumberType.DOUBLE, "m2"),
+                numberFieldType(NumberType.LONG, "m2"),
                 numberFieldType(NumberType.DOUBLE, "m3"),
         };
     }
@@ -516,11 +540,29 @@ private InternalAggregation collect(AggregationBuilder builder, Query query,
     }
 
     private InternalTopMetrics.TopMetric top(long sortValue, double... metricValues) {
-        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues);
+        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues(metricValues));
+    }
+
+    private InternalTopMetrics.TopMetric top(long sortValue, long... metricValues) {
+        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues(metricValues));
     }
 
     private InternalTopMetrics.TopMetric top(double sortValue, double... metricValues) {
-        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues);
+        return new InternalTopMetrics.TopMetric(DocValueFormat.RAW, SortValue.from(sortValue), metricValues(metricValues));
+    }
+
+    private List<InternalTopMetrics.MetricValue> metricValues(double... metricValues) {
+        return metricValues(Arrays.stream(metricValues).mapToObj(SortValue::from).toArray(SortValue[]::new));
+    }
+
+    private List<InternalTopMetrics.MetricValue> metricValues(long... metricValues) {
+        return metricValues(Arrays.stream(metricValues).mapToObj(SortValue::from).toArray(SortValue[]::new));
+    }
+
+    private List<InternalTopMetrics.MetricValue> metricValues(SortValue... metricValues) {
+        return Arrays.stream(metricValues)
+                .map(v -> new InternalTopMetrics.MetricValue(DocValueFormat.RAW, v))
+                .collect(toList());
     }
 
     /**
diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle
new file mode 100644
index 0000000000000..8aac2b9f885ac
--- /dev/null
+++ b/x-pack/plugin/async-search/build.gradle
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+esplugin {
+    name 'x-pack-async-search'
+    description 'A module which allows to track the progress of a search asynchronously.'
+    classname 'org.elasticsearch.xpack.search.AsyncSearch'
+    extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-async-search'
+
+compileJava.options.compilerArgs << "-Xlint:-rawtypes"
+compileTestJava.options.compilerArgs << "-Xlint:-rawtypes"
+
+integTest.enabled = false
+
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+  project.subprojects
+    .find { it.path == project.path + ":qa" }
+    .subprojects
+    .findAll { it.path.startsWith(project.path + ":qa") }
+    .each { check.dependsOn it.check }
+}
+
+dependencies {
+  compileOnly project(":server")
+
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('ilm'))
+}
+
+dependencyLicenses {
+  ignoreSha 'x-pack-core'
+}
diff --git a/x-pack/plugin/async-search/qa/build.gradle b/x-pack/plugin/async-search/qa/build.gradle
new file mode 100644
index 0000000000000..79ff4091f6d2d
--- /dev/null
+++ b/x-pack/plugin/async-search/qa/build.gradle
@@ -0,0 +1,8 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.build'
+test.enabled = false
+
+dependencies {
+  compile project(':test:framework')
+}
diff --git a/x-pack/plugin/async-search/qa/security/build.gradle b/x-pack/plugin/async-search/qa/security/build.gradle
new file mode 100644
index 0000000000000..0337749a9eebd
--- /dev/null
+++ b/x-pack/plugin/async-search/qa/security/build.gradle
@@ -0,0 +1,19 @@
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('async-search'), configuration: 'runtime')
+  testCompile project(':x-pack:plugin:async-search:qa')
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.security.enabled', 'true'
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user username: "test-admin", password: 'x-pack-test-password', role: "test-admin"
+  user username: "user1", password: 'x-pack-test-password', role: "user1"
+  user username: "user2", password: 'x-pack-test-password', role: "user2"
+}
diff --git a/x-pack/plugin/async-search/qa/security/roles.yml b/x-pack/plugin/async-search/qa/security/roles.yml
new file mode 100644
index 0000000000000..4ab3be5ff0571
--- /dev/null
+++ b/x-pack/plugin/async-search/qa/security/roles.yml
@@ -0,0 +1,33 @@
+# All cluster rights
+# All operations on all indices
+# Run as all users
+test-admin:
+  cluster:
+    - all
+  indices:
+    - names: '*'
+      privileges: [ all ]
+  run_as:
+    - '*'
+
+user1:
+  cluster:
+    - cluster:monitor/main
+  indices:
+    - names: ['index-user1', 'index' ]
+      privileges:
+        - read
+        - write
+        - create_index
+        - indices:admin/refresh
+
+user2:
+  cluster:
+    - cluster:monitor/main
+  indices:
+    - names: [ 'index-user2', 'index' ]
+      privileges:
+        - read
+        - write
+        - create_index
+        - indices:admin/refresh
diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
new file mode 100644
index 0000000000000..05e0132deb7d3
--- /dev/null
+++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.search;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.elasticsearch.xpack.search.AsyncSearchIndexService.INDEX;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class AsyncSearchSecurityIT extends ESRestTestCase {
+    /**
+     * All tests run as a superuser but use <code>es-security-runas-user</code> to become a less privileged user.
+     */
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("test-admin", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Before
+    public void indexDocuments() throws IOException {
+        createIndex("index", Settings.EMPTY);
+        index("index", "0", "foo", "bar");
+        refresh("index");
+
+        createIndex("index-user1", Settings.EMPTY);
+        index("index-user1", "0", "foo", "bar");
+        refresh("index-user1");
+
+        createIndex("index-user2", Settings.EMPTY);
+        index("index-user2", "0", "foo", "bar");
+        refresh("index-user2");
+    }
+
+    public void testWithUsers() throws Exception {
+        testCase("user1", "user2");
+        testCase("user2", "user1");
+    }
+
+    private void testCase(String user, String other) throws Exception {
+       for (String indexName : new String[] {"index", "index-" + user}) {
+            Response submitResp = submitAsyncSearch(indexName, "foo:bar", TimeValue.timeValueSeconds(10), user);
+            assertOK(submitResp);
+            String id = extractResponseId(submitResp);
+            Response getResp = getAsyncSearch(id, user);
+            assertOK(getResp);
+
+            // other cannot access the result
+            ResponseException exc = expectThrows(ResponseException.class, () -> getAsyncSearch(id, other));
+            assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404));
+
+            // other cannot delete the result
+            exc = expectThrows(ResponseException.class, () -> deleteAsyncSearch(id, other));
+            assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404));
+
+            // other and user cannot access the result from direct get calls
+           AsyncSearchId searchId = AsyncSearchId.decode(id);
+           for (String runAs : new String[] {user, other}) {
+               exc = expectThrows(ResponseException.class, () -> get(INDEX, searchId.getDocId(), runAs));
+               assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+               assertThat(exc.getMessage(), containsString("unauthorized"));
+           }
+
+            Response delResp = deleteAsyncSearch(id, user);
+            assertOK(delResp);
+        }
+        ResponseException exc = expectThrows(ResponseException.class,
+            () -> submitAsyncSearch("index-" + other, "*", TimeValue.timeValueSeconds(10), user));
+        assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+        assertThat(exc.getMessage(), containsString("unauthorized"));
+    }
+
+    static String extractResponseId(Response response) throws IOException {
+        Map<String, Object> map = toMap(response);
+        return (String) map.get("id");
+    }
+
+    static void index(String index, String id, Object... fields) throws IOException {
+        XContentBuilder document = jsonBuilder().startObject();
+        for (int i = 0; i < fields.length; i += 2) {
+            document.field((String) fields[i], fields[i + 1]);
+        }
+        document.endObject();
+        final Request request = new Request("POST", "/" + index + "/_doc/" + id);
+        request.setJsonEntity(Strings.toString(document));
+        assertOK(client().performRequest(request));
+    }
+
+    static void refresh(String index) throws IOException {
+        assertOK(adminClient().performRequest(new Request("POST", "/" + index + "/_refresh")));
+    }
+
+    static Response get(String index, String id, String user) throws IOException {
+        final Request request = new Request("GET", "/" + index + "/_doc/" + id);
+        setRunAsHeader(request, user);
+        return client().performRequest(request);
+    }
+
+    static Response submitAsyncSearch(String indexName, String query, TimeValue waitForCompletion, String user) throws IOException {
+        final Request request = new Request("POST", indexName + "/_async_search");
+        setRunAsHeader(request, user);
+        request.addParameter("q", query);
+        request.addParameter("wait_for_completion", waitForCompletion.toString());
+        // we do the cleanup explicitly
+        request.addParameter("clean_on_completion", "false");
+        return client().performRequest(request);
+    }
+
+    static Response getAsyncSearch(String id, String user) throws IOException {
+        final Request request = new Request("GET",  "/_async_search/" + id);
+        setRunAsHeader(request, user);
+        request.addParameter("wait_for_completion", "0ms");
+        return client().performRequest(request);
+    }
+
+    static Response deleteAsyncSearch(String id, String user) throws IOException {
+        final Request request = new Request("DELETE",  "/_async_search/" + id);
+        setRunAsHeader(request, user);
+        return client().performRequest(request);
+    }
+
+    static Map<String, Object> toMap(Response response) throws IOException {
+        return toMap(EntityUtils.toString(response.getEntity()));
+    }
+
+    static Map<String, Object> toMap(String response) {
+        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
+    }
+
+    static void setRunAsHeader(Request request, String user) {
+        final RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+        builder.addHeader(RUN_AS_USER_HEADER, user);
+        request.setOptions(builder);
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java
new file mode 100644
index 0000000000000..580f7d8a5560a
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+public final class AsyncSearch extends Plugin implements ActionPlugin {
+    private final Settings settings;
+
+    public AsyncSearch(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(
+            new ActionHandler<>(SubmitAsyncSearchAction.INSTANCE, TransportSubmitAsyncSearchAction.class),
+            new ActionHandler<>(GetAsyncSearchAction.INSTANCE, TransportGetAsyncSearchAction.class),
+            new ActionHandler<>(DeleteAsyncSearchAction.INSTANCE, TransportDeleteAsyncSearchAction.class)
+        );
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+            new RestSubmitAsyncSearchAction(),
+            new RestGetAsyncSearchAction(),
+            new RestDeleteAsyncSearchAction()
+        );
+    }
+
+    @Override
+    public Collection<Object> createComponents(Client client,
+                                               ClusterService clusterService,
+                                               ThreadPool threadPool,
+                                               ResourceWatcherService resourceWatcherService,
+                                               ScriptService scriptService,
+                                               NamedXContentRegistry xContentRegistry,
+                                               Environment environment,
+                                               NodeEnvironment nodeEnvironment,
+                                               NamedWriteableRegistry namedWriteableRegistry,
+                                               IndexNameExpressionResolver indexNameExpressionResolver) {
+        if (DiscoveryNode.isDataNode(environment.settings())) {
+            // only data nodes should be eligible to run the maintenance service.
+            AsyncSearchIndexService indexService =
+                new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, namedWriteableRegistry);
+            AsyncSearchMaintenanceService maintenanceService =
+                new AsyncSearchMaintenanceService(nodeEnvironment.nodeId(), threadPool, indexService, TimeValue.timeValueHours(1));
+            clusterService.addListener(maintenanceService);
+            return Collections.singletonList(maintenanceService);
+        } else {
+            return Collections.emptyList();
+        }
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java
new file mode 100644
index 0000000000000..ef8b47ee00312
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.TaskId;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Base64;
+import java.util.Objects;
+
+/**
+ * A class that contains all information related to a submitted async search.
+ */
+class AsyncSearchId {
+    private final String docId;
+    private final TaskId taskId;
+    private final String encoded;
+
+    AsyncSearchId(String docId, TaskId taskId) {
+        this.docId = docId;
+        this.taskId = taskId;
+        this.encoded = encode(docId, taskId);
+    }
+
+    /**
+     * The document id of the response in the index if the task is not running.
+     */
+    String getDocId() {
+        return docId;
+    }
+
+    /**
+     * The {@link TaskId} of the async search in the task manager.
+     */
+    TaskId getTaskId() {
+        return taskId;
+    }
+
+    /**
+     * Gets the encoded string that represents this search.
+     */
+    String getEncoded() {
+        return encoded;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AsyncSearchId searchId = (AsyncSearchId) o;
+        return docId.equals(searchId.docId) &&
+            taskId.equals(searchId.taskId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(docId, taskId);
+    }
+
+    @Override
+    public String toString() {
+        return "AsyncSearchId{" +
+            "docId='" + docId + '\'' +
+            ", taskId=" + taskId +
+            '}';
+    }
+
+    /**
+     * Encodes the informations needed to retrieve a async search response
+     * in a base64 encoded string.
+     */
+    static String encode(String docId, TaskId taskId) {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeString(docId);
+            out.writeString(taskId.toString());
+            return Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(out.bytes()));
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    /**
+     * Decodes a base64 encoded string into an {@link AsyncSearchId} that can be used
+     * to retrieve the response of an async search.
+     */
+    static AsyncSearchId decode(String id) {
+        final AsyncSearchId searchId;
+        try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getUrlDecoder().decode(id)))) {
+            searchId = new AsyncSearchId(in.readString(), new TaskId(in.readString()));
+            if (in.available() > 0) {
+                throw new IllegalArgumentException("invalid id:[" + id + "]");
+            }
+        } catch (IOException e) {
+            throw new IllegalArgumentException("invalid id:[" + id + "]");
+        }
+        return searchId;
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
new file mode 100644
index 0000000000000..d1ec9fac7c06f
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
@@ -0,0 +1,342 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.OriginSettingClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
+import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY;
+
+/**
+ * A service that exposes the CRUD operations for the async-search index.
+ */
+class AsyncSearchIndexService {
+    private static final Logger logger = LogManager.getLogger(AsyncSearchIndexService.class);
+
+    public static final String INDEX = ".async-search";
+
+    public static final String HEADERS_FIELD = "headers";
+    public static final String EXPIRATION_TIME_FIELD = "expiration_time";
+    public static final String RESULT_FIELD = "result";
+
+    public static Settings settings() {
+        return Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .build();
+    }
+
+    public static XContentBuilder mappings() throws IOException {
+        XContentBuilder builder = jsonBuilder()
+            .startObject()
+                .startObject(SINGLE_MAPPING_NAME)
+                    .startObject("_meta")
+                        .field("version", Version.CURRENT)
+                    .endObject()
+                    .field("dynamic", "strict")
+                    .startObject("properties")
+                        .startObject(HEADERS_FIELD)
+                            .field("type", "object")
+                            .field("enabled", "false")
+                        .endObject()
+                        .startObject(RESULT_FIELD)
+                            .field("type", "object")
+                            .field("enabled", "false")
+                        .endObject()
+                        .startObject(EXPIRATION_TIME_FIELD)
+                            .field("type", "long")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        return builder;
+    }
+
+    private final ClusterService clusterService;
+    private final Client client;
+    private final SecurityContext securityContext;
+    private final NamedWriteableRegistry registry;
+
+    AsyncSearchIndexService(ClusterService clusterService,
+                            ThreadContext threadContext,
+                            Client client,
+                            NamedWriteableRegistry registry) {
+        this.clusterService = clusterService;
+        this.securityContext = new SecurityContext(clusterService.getSettings(), threadContext);
+        this.client = new OriginSettingClient(client, ASYNC_SEARCH_ORIGIN);
+        this.registry = registry;
+    }
+
+    /**
+     * Returns the internal client with origin.
+     */
+    Client getClient() {
+        return client;
+    }
+
+    /**
+     * Creates the index with the expected settings and mappings if it doesn't exist.
+     */
+    void createIndexIfNecessary(ActionListener<Void> listener) {
+        if (clusterService.state().routingTable().hasIndex(AsyncSearchIndexService.INDEX) == false) {
+            try {
+                client.admin().indices().prepareCreate(INDEX)
+                    .setSettings(settings())
+                    .setMapping(mappings())
+                    .execute(ActionListener.wrap(
+                        resp -> listener.onResponse(null),
+                        exc -> {
+                            if (ExceptionsHelper.unwrapCause(exc) instanceof ResourceAlreadyExistsException) {
+                                listener.onResponse(null);
+                            } else {
+                                logger.error("failed to create async-search index", exc);
+                                listener.onFailure(exc);
+                            }
+                        }));
+            } catch (Exception exc) {
+                logger.error("failed to create async-search index", exc);
+                listener.onFailure(exc);
+            }
+        } else {
+            listener.onResponse(null);
+        }
+    }
+
+    /**
+     * Stores the initial response with the original headers of the authenticated user
+     * and the expected expiration time.
+     */
+    void storeInitialResponse(String docId,
+                              Map<String, String> headers,
+                              AsyncSearchResponse response,
+                              ActionListener<IndexResponse> listener) throws IOException {
+        Map<String, Object> source = new HashMap<>();
+        source.put(HEADERS_FIELD, headers);
+        source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime());
+        source.put(RESULT_FIELD, encodeResponse(response));
+        IndexRequest indexRequest = new IndexRequest(INDEX)
+            .create(true)
+            .id(docId)
+            .source(source, XContentType.JSON);
+        createIndexIfNecessary(ActionListener.wrap(v -> client.index(indexRequest, listener), listener::onFailure));
+    }
+
+    /**
+     * Stores the final response if the place-holder document is still present (update).
+     */
+    void storeFinalResponse(String docId,
+                            AsyncSearchResponse response,
+                            ActionListener<UpdateResponse> listener) throws IOException {
+        Map<String, Object> source = new HashMap<>();
+        source.put(RESULT_FIELD, encodeResponse(response));
+        UpdateRequest request = new UpdateRequest()
+            .index(INDEX)
+            .id(docId)
+            .doc(source, XContentType.JSON);
+        createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure));
+    }
+
+    /**
+     * Updates the expiration time of the provided <code>docId</code> if the place-holder
+     * document is still present (update).
+     */
+    void updateExpirationTime(String docId,
+                              long expirationTimeMillis,
+                              ActionListener<UpdateResponse> listener) {
+        Map<String, Object> source = Collections.singletonMap(EXPIRATION_TIME_FIELD, expirationTimeMillis);
+        UpdateRequest request = new UpdateRequest().index(INDEX)
+            .id(docId)
+            .doc(source, XContentType.JSON);
+        createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure));
+    }
+
+    /**
+     * Deletes the provided <code>searchId</code> from the index if present.
+     */
+    void deleteResponse(AsyncSearchId searchId,
+                        boolean failIfNotFound,
+                        ActionListener<AcknowledgedResponse> listener) {
+        DeleteRequest request = new DeleteRequest(INDEX).id(searchId.getDocId());
+        createIndexIfNecessary(
+            ActionListener.wrap(v -> client.delete(request,
+                ActionListener.wrap(
+                    resp -> {
+                        if (resp.status() == RestStatus.NOT_FOUND && failIfNotFound) {
+                            listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
+                        } else {
+                            listener.onResponse(new AcknowledgedResponse(true));
+                        }
+                    },
+                    exc -> {
+                        logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc);
+                        listener.onFailure(exc);
+                    })),
+                listener::onFailure));
+    }
+
+    /**
+     * Returns the {@link AsyncSearchTask} if the provided <code>searchId</code>
+     * is registered in the task manager, <code>null</code> otherwise.
+     *
+     * This method throws a {@link ResourceNotFoundException} if the authenticated user
+     * is not the creator of the original task.
+     */
+    AsyncSearchTask getTask(TaskManager taskManager, AsyncSearchId searchId) throws IOException {
+        Task task = taskManager.getTask(searchId.getTaskId().getId());
+        if (task instanceof AsyncSearchTask == false) {
+            return null;
+        }
+        AsyncSearchTask searchTask = (AsyncSearchTask) task;
+        if (searchTask.getSearchId().equals(searchId) == false) {
+            return null;
+        }
+
+        // Check authentication for the user
+        final Authentication auth = securityContext.getAuthentication();
+        if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) {
+            throw new ResourceNotFoundException(searchId.getEncoded() + " not found");
+        }
+        return searchTask;
+    }
+
+    /**
+     * Gets the response from the index if present, or delegate a {@link ResourceNotFoundException}
+     * failure to the provided listener if not.
+     */
+    void getResponse(AsyncSearchId searchId,
+                     ActionListener<AsyncSearchResponse> listener) {
+        final Authentication current = securityContext.getAuthentication();
+        GetRequest internalGet = new GetRequest(INDEX)
+            .preference(searchId.getEncoded())
+            .id(searchId.getDocId());
+        client.get(internalGet, ActionListener.wrap(
+            get -> {
+                if (get.isExists() == false) {
+                    listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
+                    return;
+                }
+
+                // check the authentication of the current user against the user that initiated the async search
+                @SuppressWarnings("unchecked")
+                Map<String, String> headers = (Map<String, String>) get.getSource().get(HEADERS_FIELD);
+                if (ensureAuthenticatedUserIsSame(headers, current) == false) {
+                    listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
+                    return;
+                }
+
+                String encoded = (String) get.getSource().get(RESULT_FIELD);
+                listener.onResponse(encoded != null ? decodeResponse(encoded) : null);
+            },
+            listener::onFailure
+        ));
+    }
+
+    /**
+     * Extracts the authentication from the original headers and checks that it matches
+     * the current user. This function returns always <code>true</code> if the provided
+     * <code>headers</code> do not contain any authentication.
+     */
+    boolean ensureAuthenticatedUserIsSame(Map<String, String> originHeaders, Authentication current) throws IOException {
+        if (originHeaders == null || originHeaders.containsKey(AUTHENTICATION_KEY) == false) {
+            // no authorization attached to the original request
+            return true;
+        }
+        if (current == null) {
+            // origin is an authenticated user but current is not
+            return false;
+        }
+        Authentication origin = AuthenticationContextSerializer.decode(originHeaders.get(AUTHENTICATION_KEY));
+        return ensureAuthenticatedUserIsSame(origin, current);
+    }
+
+    /**
+     * Compares the {@link Authentication} that was used to create the {@link AsyncSearchId} with the
+     * current authentication.
+     */
+    boolean ensureAuthenticatedUserIsSame(Authentication original, Authentication current) {
+        final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal());
+        final boolean sameRealmType;
+        if (original.getUser().isRunAs()) {
+            if (current.getUser().isRunAs()) {
+                sameRealmType = original.getLookedUpBy().getType().equals(current.getLookedUpBy().getType());
+            }  else {
+                sameRealmType = original.getLookedUpBy().getType().equals(current.getAuthenticatedBy().getType());
+            }
+        } else if (current.getUser().isRunAs()) {
+            sameRealmType = original.getAuthenticatedBy().getType().equals(current.getLookedUpBy().getType());
+        } else {
+            sameRealmType = original.getAuthenticatedBy().getType().equals(current.getAuthenticatedBy().getType());
+        }
+        return samePrincipal && sameRealmType;
+    }
+
+    /**
+     * Encode the provided response in a binary form using base64 encoding.
+     */
+    String encodeResponse(AsyncSearchResponse response) throws IOException {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            Version.writeVersion(Version.CURRENT, out);
+            response.writeTo(out);
+            return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes()));
+        }
+    }
+
+    /**
+     * Decode the provided base-64 bytes into a {@link AsyncSearchResponse}.
+     */
+    AsyncSearchResponse decodeResponse(String value) throws IOException {
+        try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getDecoder().decode(value)))) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(buf, registry)) {
+                in.setVersion(Version.readVersion(in));
+                return new AsyncSearchResponse(in);
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java
new file mode 100644
index 0000000000000..1cac9c0eaf458
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.search;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.DeleteByQueryAction;
+import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.elasticsearch.threadpool.Scheduler;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.xpack.search.AsyncSearchIndexService.EXPIRATION_TIME_FIELD;
+
+/**
+ * A service that runs a periodic cleanup over the async-search index.
+ */
+class AsyncSearchMaintenanceService implements Releasable, ClusterStateListener {
+    private static final Logger logger = LogManager.getLogger(AsyncSearchMaintenanceService.class);
+
+    private final String localNodeId;
+    private final ThreadPool threadPool;
+    private final AsyncSearchIndexService indexService;
+    private final TimeValue delay;
+
+    private final AtomicBoolean isCleanupRunning = new AtomicBoolean(false);
+    private final AtomicBoolean isClosed = new AtomicBoolean(false);
+    private volatile Scheduler.Cancellable cancellable;
+
+    AsyncSearchMaintenanceService(String localNodeId,
+                                  ThreadPool threadPool,
+                                  AsyncSearchIndexService indexService,
+                                  TimeValue delay) {
+        this.localNodeId = localNodeId;
+        this.threadPool = threadPool;
+        this.indexService = indexService;
+        this.delay = delay;
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        final ClusterState state = event.state();
+        if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            // Wait until the gateway has recovered from disk.
+            return;
+        }
+        tryStartCleanup(state);
+    }
+
+    void tryStartCleanup(ClusterState state) {
+        if (isClosed.get()) {
+            return;
+        }
+        IndexRoutingTable indexRouting = state.routingTable().index(AsyncSearchIndexService.INDEX);
+        if (indexRouting == null) {
+            if (isCleanupRunning.compareAndSet(true, false)) {
+                close();
+            }
+            return;
+        }
+        String primaryNodeId = indexRouting.shard(0).primaryShard().currentNodeId();
+        if (localNodeId.equals(primaryNodeId)) {
+            if (isCleanupRunning.compareAndSet(false, true)) {
+                executeNextCleanup();
+            }
+        } else if (isCleanupRunning.compareAndSet(true, false)) {
+            close();
+        }
+    }
+
+    synchronized void executeNextCleanup() {
+        if (isClosed.get() == false && isCleanupRunning.get()) {
+            long nowInMillis = System.currentTimeMillis();
+            DeleteByQueryRequest toDelete = new DeleteByQueryRequest()
+                .setQuery(QueryBuilders.rangeQuery(EXPIRATION_TIME_FIELD).lte(nowInMillis));
+            indexService.getClient()
+                .execute(DeleteByQueryAction.INSTANCE, toDelete, ActionListener.wrap(() -> scheduleNextCleanup()));
+        }
+    }
+
+    synchronized void scheduleNextCleanup() {
+        if (isClosed.get() == false && isCleanupRunning.get()) {
+            try {
+                cancellable = threadPool.schedule(this::executeNextCleanup, delay, ThreadPool.Names.GENERIC);
+            } catch (EsRejectedExecutionException e) {
+                if (e.isExecutorShutdown()) {
+                    logger.debug("failed to schedule next maintenance task; shutting down", e);
+                } else {
+                    throw e;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+        if (cancellable != null && cancellable.isCancelled() == false) {
+            cancellable.cancel();
+        }
+        isClosed.compareAndSet(false, true);
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
new file mode 100644
index 0000000000000..cb2862a087774
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
@@ -0,0 +1,370 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
+import org.elasticsearch.action.search.SearchProgressActionListener;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchResponse.Clusters;
+import org.elasticsearch.action.search.SearchShard;
+import org.elasticsearch.action.search.SearchTask;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.threadpool.Scheduler.Cancellable;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+/**
+ * Task that tracks the progress of a currently running {@link SearchRequest}.
+ */
+final class AsyncSearchTask extends SearchTask {
+    private final AsyncSearchId searchId;
+    private final Client client;
+    private final ThreadPool threadPool;
+    private final Supplier<ReduceContext> reduceContextSupplier;
+    private final Listener progressListener;
+
+    private final Map<String, String> originHeaders;
+
+    private boolean hasInitialized;
+    private boolean hasCompleted;
+    private long completionId;
+    private final List<Runnable> initListeners = new ArrayList<>();
+    private final Map<Long, Consumer<AsyncSearchResponse>> completionListeners = new HashMap<>();
+
+    private volatile long expirationTimeMillis;
+    private final AtomicBoolean isCancelling = new AtomicBoolean(false);
+
+    private AtomicReference<MutableSearchResponse> searchResponse;
+
+    /**
+     * Creates an instance of {@link AsyncSearchTask}.
+     *
+     * @param id The id of the task.
+     * @param type The type of the task.
+     * @param action The action name.
+     * @param parentTaskId The parent task id.
+     * @param originHeaders All the request context headers.
+     * @param taskHeaders The filtered request headers for the task.
+     * @param searchId The {@link AsyncSearchId} of the task.
+     * @param threadPool The threadPool to schedule runnable.
+     * @param reduceContextSupplier A supplier to create final reduce contexts.
+     */
+    AsyncSearchTask(long id,
+                    String type,
+                    String action,
+                    TaskId parentTaskId,
+                    TimeValue keepAlive,
+                    Map<String, String> originHeaders,
+                    Map<String, String> taskHeaders,
+                    AsyncSearchId searchId,
+                    Client client,
+                    ThreadPool threadPool,
+                    Supplier<ReduceContext> reduceContextSupplier) {
+        super(id, type, action, "async_search", parentTaskId, taskHeaders);
+        this.expirationTimeMillis = getStartTime() + keepAlive.getMillis();
+        this.originHeaders = originHeaders;
+        this.searchId = searchId;
+        this.client = client;
+        this.threadPool = threadPool;
+        this.reduceContextSupplier = reduceContextSupplier;
+        this.progressListener = new Listener();
+        this.searchResponse = new AtomicReference<>();
+        setProgressListener(progressListener);
+    }
+
+    /**
+     * Returns all of the request contexts headers
+     */
+    Map<String, String> getOriginHeaders() {
+        return originHeaders;
+    }
+
+    /**
+     * Returns the {@link AsyncSearchId} of the task
+     */
+    AsyncSearchId getSearchId() {
+        return searchId;
+    }
+
+    Listener getSearchProgressActionListener() {
+        return progressListener;
+    }
+
+    /**
+     * Update the expiration time of the (partial) response.
+     */
+    public void setExpirationTime(long expirationTimeMillis) {
+        this.expirationTimeMillis = expirationTimeMillis;
+    }
+
+    /**
+     * Cancels the running task and its children.
+     */
+    public void cancelTask(Runnable runnable) {
+        if (isCancelled() == false && isCancelling.compareAndSet(false, true)) {
+            CancelTasksRequest req = new CancelTasksRequest().setTaskId(searchId.getTaskId());
+            client.admin().cluster().cancelTasks(req, new ActionListener<>() {
+                @Override
+                public void onResponse(CancelTasksResponse cancelTasksResponse) {
+                    runnable.run();
+                }
+
+                @Override
+                public void onFailure(Exception exc) {
+                    // cancelling failed
+                    isCancelling.compareAndSet(true, false);
+                    runnable.run();
+                }
+            });
+        } else {
+            runnable.run();
+       }
+    }
+
+    @Override
+    protected void onCancelled() {
+        super.onCancelled();
+        isCancelling.compareAndSet(true, false);
+    }
+
+    /**
+     * Creates a listener that listens for an {@link AsyncSearchResponse} and executes the
+     * consumer when the task is finished or when the provided <code>waitForCompletion</code>
+     * timeout occurs. In such case the consumed {@link AsyncSearchResponse} will contain partial results.
+     */
+    public void addCompletionListener(ActionListener<AsyncSearchResponse> listener, TimeValue waitForCompletion) {
+        boolean executeImmediately = false;
+        long startTime = threadPool.relativeTimeInMillis();
+        synchronized (this) {
+            if (hasCompleted) {
+                executeImmediately = true;
+            } else {
+                addInitListener(() -> {
+                    final TimeValue remainingWaitForCompletion;
+                    if (waitForCompletion.getMillis() > 0) {
+                        long elapsedTime = threadPool.relativeTimeInMillis() - startTime;
+                        // subtract the initialization time from the provided waitForCompletion.
+                        remainingWaitForCompletion = TimeValue.timeValueMillis(Math.max(0, waitForCompletion.getMillis() - elapsedTime));
+                    } else {
+                        remainingWaitForCompletion = TimeValue.ZERO;
+                    }
+                    internalAddCompletionListener(listener, remainingWaitForCompletion);
+                });
+            }
+        }
+        if (executeImmediately) {
+            listener.onResponse(getResponse());
+        }
+    }
+
+    /**
+     * Creates a listener that listens for an {@link AsyncSearchResponse} and executes the
+     * consumer when the task is finished.
+     */
+    public void addCompletionListener(Consumer<AsyncSearchResponse>  listener) {
+        boolean executeImmediately = false;
+        synchronized (this) {
+            if (hasCompleted) {
+                executeImmediately = true;
+            } else {
+                completionListeners.put(completionId++, listener::accept);
+            }
+        }
+        if (executeImmediately) {
+            listener.accept(getResponse());
+        }
+    }
+
+    private void internalAddCompletionListener(ActionListener<AsyncSearchResponse> listener, TimeValue waitForCompletion) {
+        boolean executeImmediately = false;
+        synchronized (this) {
+            if (hasCompleted || waitForCompletion.getMillis() == 0) {
+                executeImmediately = true;
+            } else {
+                // ensure that we consumes the listener only once
+                AtomicBoolean hasRun = new AtomicBoolean(false);
+                long id = completionId++;
+
+                final Cancellable cancellable;
+                try {
+                    cancellable = threadPool.schedule(() -> {
+                        if (hasRun.compareAndSet(false, true)) {
+                            // timeout occurred before completion
+                            removeCompletionListener(id);
+                            listener.onResponse(getResponse());
+                        }
+                    }, waitForCompletion, "generic");
+                } catch (EsRejectedExecutionException exc) {
+                    listener.onFailure(exc);
+                    return;
+                }
+                completionListeners.put(id, resp -> {
+                    if (hasRun.compareAndSet(false, true)) {
+                        // completion occurred before timeout
+                        cancellable.cancel();
+                        listener.onResponse(resp);
+                    }
+                });
+            }
+        }
+        if (executeImmediately) {
+            listener.onResponse(getResponse());
+        }
+    }
+
+    private void removeCompletionListener(long id) {
+        synchronized (this) {
+            if (hasCompleted == false) {
+                completionListeners.remove(id);
+            }
+        }
+    }
+
+    private void addInitListener(Runnable listener) {
+        boolean executeImmediately = false;
+        synchronized (this) {
+            if (hasInitialized) {
+                executeImmediately = true;
+            } else {
+                initListeners.add(listener);
+            }
+        }
+        if (executeImmediately) {
+            listener.run();
+        }
+    }
+
+    private void executeInitListeners() {
+        synchronized (this) {
+            if (hasInitialized) {
+                return;
+            }
+            hasInitialized = true;
+        }
+        for (Runnable listener : initListeners) {
+            listener.run();
+        }
+        initListeners.clear();
+    }
+
+    private void executeCompletionListeners() {
+        synchronized (this) {
+            if (hasCompleted) {
+                return;
+            }
+            hasCompleted = true;
+        }
+        AsyncSearchResponse finalResponse = getResponse();
+        for (Consumer<AsyncSearchResponse> listener : completionListeners.values()) {
+            listener.accept(finalResponse);
+        }
+        completionListeners.clear();
+    }
+
+    private AsyncSearchResponse getResponse() {
+        assert searchResponse.get() != null;
+        return searchResponse.get().toAsyncSearchResponse(this, expirationTimeMillis);
+    }
+
+    // cancels the task if it expired
+    private void checkExpiration() {
+        long now = System.currentTimeMillis();
+        if (expirationTimeMillis < now) {
+            cancelTask(() -> {});
+        }
+    }
+
+    class Listener extends SearchProgressActionListener {
+        @Override
+        protected void onQueryResult(int shardIndex) {
+            checkExpiration();
+        }
+
+        @Override
+        protected void onFetchResult(int shardIndex) {
+            checkExpiration();
+        }
+
+        @Override
+        protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
+            // best effort to cancel expired tasks
+            checkExpiration();
+            searchResponse.get().addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget));
+        }
+
+        @Override
+        protected void onFetchFailure(int shardIndex, Exception exc) {
+            checkExpiration();
+        }
+
+        @Override
+        protected void onListShards(List<SearchShard> shards, List<SearchShard> skipped, Clusters clusters, boolean fetchPhase) {
+            // best effort to cancel expired tasks
+            checkExpiration();
+            searchResponse.compareAndSet(null,
+                new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier));
+            executeInitListeners();
+        }
+
+        @Override
+        public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+            // best effort to cancel expired tasks
+            checkExpiration();
+            searchResponse.get().updatePartialResponse(shards.size(),
+                new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
+                    null, null, false, null, reducePhase), aggs == null);
+        }
+
+        @Override
+        public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+            // best effort to cancel expired tasks
+            checkExpiration();
+            searchResponse.get().updatePartialResponse(shards.size(),
+                new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
+                    null, null, false, null, reducePhase), true);
+        }
+
+        @Override
+        public void onResponse(SearchResponse response) {
+            searchResponse.get().updateFinalResponse(response.getSuccessfulShards(), response.getInternalResponse());
+            executeCompletionListeners();
+        }
+
+        @Override
+        public void onFailure(Exception exc) {
+            if (searchResponse.get() == null) {
+                // if the failure occurred before calling onListShards
+                searchResponse.compareAndSet(null,
+                    new MutableSearchResponse(-1, -1, null, reduceContextSupplier));
+            }
+            searchResponse.get().updateWithFailure(exc);
+            executeInitListeners();
+            executeCompletionListeners();
+        }
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
new file mode 100644
index 0000000000000..1949aceb6bd91
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchResponse.Clusters;
+import org.elasticsearch.action.search.SearchResponseSections;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static java.util.Collections.singletonList;
+import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
+import static org.elasticsearch.search.aggregations.InternalAggregations.topLevelReduce;
+
+/**
+ * A mutable search response that allows to update and create partial response synchronously.
+ * Synchronized methods ensure that updates of the content are blocked if another thread is
+ * creating an async response concurrently. This limits the number of final reduction that can
+ * run concurrently to 1 and ensures that we pause the search progress when an {@link AsyncSearchResponse} is built.
+ */
+class MutableSearchResponse {
+    private final int totalShards;
+    private final int skippedShards;
+    private final Clusters clusters;
+    private final AtomicArray<ShardSearchFailure> shardFailures;
+    private final Supplier<ReduceContext> reduceContextSupplier;
+
+    private int version;
+    private boolean isPartial;
+    private boolean isFinalReduce;
+    private int successfulShards;
+    private SearchResponseSections sections;
+    private ElasticsearchException failure;
+
+    private boolean frozen;
+
+    /**
+     * Creates a new mutable search response.
+     *
+     * @param totalShards The number of shards that participate in the request, or -1 to indicate a failure.
+     * @param skippedShards The number of skipped shards, or -1 to indicate a failure.
+     * @param clusters The remote clusters statistics.
+     * @param reduceContextSupplier A supplier to run final reduce on partial aggregations.
+     */
+    MutableSearchResponse(int totalShards, int skippedShards, Clusters clusters, Supplier<ReduceContext> reduceContextSupplier) {
+        this.totalShards = totalShards;
+        this.skippedShards = skippedShards;
+        this.clusters = clusters;
+        this.reduceContextSupplier = reduceContextSupplier;
+        this.version = 0;
+        this.shardFailures = totalShards == -1 ? null : new AtomicArray<>(totalShards-skippedShards);
+        this.isPartial = true;
+        this.sections = totalShards == -1 ? null : new InternalSearchResponse(
+            new SearchHits(SearchHits.EMPTY, new TotalHits(0, GREATER_THAN_OR_EQUAL_TO), Float.NaN),
+            null, null, null, false, null, 0);
+    }
+
+    /**
+     * Updates the response with the partial {@link SearchResponseSections} merged from #<code>successfulShards</code>
+     * shards.
+     */
+    synchronized void updatePartialResponse(int successfulShards, SearchResponseSections newSections, boolean isFinalReduce) {
+        failIfFrozen();
+        if (newSections.getNumReducePhases() < sections.getNumReducePhases()) {
+            // should never happen since partial response are updated under a lock
+            // in the search phase controller
+            throw new IllegalStateException("received partial response out of order: "
+                + newSections.getNumReducePhases() + " < " + sections.getNumReducePhases());
+        }
+        ++ version;
+        this.successfulShards = successfulShards;
+        this.sections = newSections;
+        this.isPartial = true;
+        this.isFinalReduce = isFinalReduce;
+    }
+
+    /**
+     * Updates the response with the final {@link SearchResponseSections} merged from #<code>successfulShards</code>
+     * shards.
+     */
+    synchronized void updateFinalResponse(int successfulShards, SearchResponseSections newSections) {
+        failIfFrozen();
+        ++ version;
+        this.successfulShards = successfulShards;
+        this.sections = newSections;
+        this.isPartial = false;
+        this.isFinalReduce = true;
+        this.frozen = true;
+    }
+
+    /**
+     * Updates the response with a fatal failure. This method preserves the partial response
+     * received from previous updates
+     */
+    synchronized void updateWithFailure(Exception exc) {
+        failIfFrozen();
+        ++ version;
+        this.isPartial = true;
+        this.failure = ElasticsearchException.guessRootCauses(exc)[0];
+        this.frozen = true;
+    }
+
+    /**
+     * Adds a shard failure concurrently (non-blocking).
+     */
+    void addShardFailure(int shardIndex, ShardSearchFailure failure) {
+        synchronized (this) {
+            failIfFrozen();
+        }
+        shardFailures.set(shardIndex, failure);
+    }
+
+    /**
+     * Creates an {@link AsyncSearchResponse} based on the current state of the mutable response.
+     * The final reduce of the aggregations is executed if needed (partial response).
+     * This method is synchronized to ensure that we don't perform final reduces concurrently.
+     */
+    synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, long expirationTime) {
+        final SearchResponse resp;
+        if (totalShards != -1) {
+            if (sections.aggregations() != null && isFinalReduce == false) {
+                InternalAggregations oldAggs = (InternalAggregations) sections.aggregations();
+                InternalAggregations newAggs = topLevelReduce(singletonList(oldAggs), reduceContextSupplier.get());
+                sections = new InternalSearchResponse(sections.hits(), newAggs, sections.suggest(),
+                    null, sections.timedOut(), sections.terminatedEarly(), sections.getNumReducePhases());
+                isFinalReduce = true;
+            }
+            long tookInMillis = TimeValue.timeValueNanos(System.nanoTime() - task.getStartTimeNanos()).getMillis();
+            resp = new SearchResponse(sections, null, totalShards, successfulShards,
+                skippedShards, tookInMillis, buildShardFailures(), clusters);
+        } else {
+            resp = null;
+        }
+        return new AsyncSearchResponse(task.getSearchId().getEncoded(), version, resp, failure, isPartial,
+            frozen == false, task.getStartTime(), expirationTime);
+    }
+
+    private void failIfFrozen() {
+        if (frozen) {
+            throw new IllegalStateException("invalid update received after the completion of the request");
+        }
+    }
+
+    private ShardSearchFailure[] buildShardFailures() {
+        if (shardFailures == null) {
+            return new ShardSearchFailure[0];
+        }
+        List<ShardSearchFailure> failures = new ArrayList<>();
+        for (int i = 0; i < shardFailures.length(); i++) {
+            ShardSearchFailure failure = shardFailures.get(i);
+            if (failure != null) {
+                failures.add(failure);
+            }
+        }
+        return failures.toArray(ShardSearchFailure[]::new);
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java
new file mode 100644
index 0000000000000..faab51dc3af73
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestHandler.Route;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+
+public class RestDeleteAsyncSearchAction extends BaseRestHandler  {
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(
+            new Route(DELETE, "/_async_search/{id}")));
+    }
+
+    @Override
+    public String getName() {
+        return "async_search_delete_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        DeleteAsyncSearchAction.Request delete = new DeleteAsyncSearchAction.Request(request.param("id"));
+        return channel -> client.execute(DeleteAsyncSearchAction.INSTANCE, delete, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
new file mode 100644
index 0000000000000..8dc29bf071e0d
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+
+import java.util.List;
+import java.util.Set;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.search.RestSubmitAsyncSearchAction.RESPONSE_PARAMS;
+
+public class RestGetAsyncSearchAction extends BaseRestHandler  {
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(new Route(GET, "/_async_search/{id}")));
+    }
+
+
+    @Override
+    public String getName() {
+        return "async_search_get_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(request.param("id"));
+        if (request.hasParam("wait_for_completion")) {
+            get.setWaitForCompletion(request.paramAsTime("wait_for_completion", get.getWaitForCompletion()));
+        }
+        if (request.hasParam("keep_alive")) {
+            get.setKeepAlive(request.paramAsTime("keep_alive", get.getKeepAlive()));
+        }
+        return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel));
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
new file mode 100644
index 0000000000000..e1020f8e5a234
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.function.IntConsumer;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.action.search.RestSearchAction.parseSearchRequest;
+
+public final class RestSubmitAsyncSearchAction extends BaseRestHandler {
+    static final String TYPED_KEYS_PARAM = "typed_keys";
+    static final Set<String> RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM);
+
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(
+            new Route(POST, "/_async_search"),
+            new Route(POST, "/{index}/_async_search")));
+    }
+
+    @Override
+    public String getName() {
+        return "async_search_submit_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        SubmitAsyncSearchRequest submit = new SubmitAsyncSearchRequest();
+        IntConsumer setSize = size -> submit.getSearchRequest().source().size(size);
+        request.withContentOrSourceParamParserOrNull(parser ->
+            parseSearchRequest(submit.getSearchRequest(), request, parser, setSize));
+
+        if (request.hasParam("wait_for_completion")) {
+            submit.setWaitForCompletion(request.paramAsTime("wait_for_completion", submit.getWaitForCompletion()));
+        }
+        if (request.hasParam("keep_alive")) {
+            submit.setKeepAlive(request.paramAsTime("keep_alive", submit.getKeepAlive()));
+        }
+        if (request.hasParam("clean_on_completion")) {
+            submit.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", submit.isCleanOnCompletion()));
+        }
+        return channel -> {
+            RestStatusToXContentListener<AsyncSearchResponse> listener = new RestStatusToXContentListener<>(channel);
+            RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel());
+            cancelClient.execute(SubmitAsyncSearchAction.INSTANCE, submit, listener);
+        };
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
new file mode 100644
index 0000000000000..14bf4eafec517
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+
+import java.io.IOException;
+
+public class TransportDeleteAsyncSearchAction extends HandledTransportAction<DeleteAsyncSearchAction.Request, AcknowledgedResponse> {
+    private final ClusterService clusterService;
+    private final TransportService transportService;
+    private final AsyncSearchIndexService store;
+
+    @Inject
+    public TransportDeleteAsyncSearchAction(TransportService transportService,
+                                            ActionFilters actionFilters,
+                                            ClusterService clusterService,
+                                            ThreadPool threadPool,
+                                            NamedWriteableRegistry registry,
+                                            Client client) {
+        super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new);
+        this.store = new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, registry);
+        this.clusterService = clusterService;
+        this.transportService = transportService;
+    }
+
+    @Override
+    protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, ActionListener<AcknowledgedResponse> listener) {
+        try {
+            AsyncSearchId searchId = AsyncSearchId.decode(request.getId());
+            DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId());
+            if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) {
+                cancelTaskAndDeleteResult(searchId, listener);
+            } else {
+                TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
+                transportService.sendRequest(node, DeleteAsyncSearchAction.NAME, request, builder.build(),
+                    new ActionListenerResponseHandler<>(listener, AcknowledgedResponse::new, ThreadPool.Names.SAME));
+            }
+        } catch (Exception exc) {
+            listener.onFailure(exc);
+        }
+    }
+
+    private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener<AcknowledgedResponse> listener) throws IOException {
+        AsyncSearchTask task = store.getTask(taskManager, searchId);
+        if (task != null) {
+            task.cancelTask(() -> store.deleteResponse(searchId, false, listener));
+        } else {
+            // the task is not running anymore so we throw a not found exception if
+            // the search id is also not present in the index (already deleted) or if the user
+            // is not allowed to access it.
+            store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResponse(searchId, true, listener), listener::onFailure));
+        }
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
new file mode 100644
index 0000000000000..01c807b19aea1
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.index.engine.DocumentMissingException;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+
+public class TransportGetAsyncSearchAction extends HandledTransportAction<GetAsyncSearchAction.Request, AsyncSearchResponse> {
+    private final Logger logger = LogManager.getLogger(TransportGetAsyncSearchAction.class);
+    private final ClusterService clusterService;
+    private final TransportService transportService;
+    private final AsyncSearchIndexService store;
+
+    @Inject
+    public TransportGetAsyncSearchAction(TransportService transportService,
+                                         ActionFilters actionFilters,
+                                         ClusterService clusterService,
+                                         NamedWriteableRegistry registry,
+                                         Client client,
+                                         ThreadPool threadPool) {
+        super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncSearchAction.Request::new);
+        this.clusterService = clusterService;
+        this.transportService = transportService;
+        this.store = new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, registry);
+    }
+
+    @Override
+    protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener<AsyncSearchResponse> listener) {
+        try {
+            long nowInMillis = System.currentTimeMillis();
+            AsyncSearchId searchId = AsyncSearchId.decode(request.getId());
+            DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId());
+            if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) {
+                if (request.getKeepAlive().getMillis() > 0) {
+                    long expirationTime = nowInMillis + request.getKeepAlive().getMillis();
+                    store.updateExpirationTime(searchId.getDocId(), expirationTime,
+                        ActionListener.wrap(
+                            p -> getSearchResponseFromTask(searchId, request, nowInMillis, expirationTime, listener),
+                            exc -> {
+                                if (exc.getCause() instanceof DocumentMissingException == false) {
+                                    logger.error("failed to retrieve " + searchId.getEncoded(), exc);
+                                }
+                                listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
+                            }
+                        ));
+                } else {
+                    getSearchResponseFromTask(searchId, request, nowInMillis, -1, listener);
+                }
+            } else {
+                TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
+                transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(),
+                    new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME));
+            }
+        } catch (Exception exc) {
+            listener.onFailure(exc);
+        }
+    }
+
+    private void getSearchResponseFromTask(AsyncSearchId searchId,
+                                           GetAsyncSearchAction.Request request,
+                                           long nowInMillis,
+                                           long expirationTimeMillis,
+                                           ActionListener<AsyncSearchResponse> listener) {
+        try {
+            final AsyncSearchTask task = store.getTask(taskManager, searchId);
+            if (task == null) {
+                getSearchResponseFromIndex(searchId, request, nowInMillis, listener);
+                return;
+            }
+
+            if (task.isCancelled()) {
+                listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
+                return;
+            }
+
+            if (expirationTimeMillis != -1) {
+                task.setExpirationTime(expirationTimeMillis);
+            }
+            task.addCompletionListener(new ActionListener<>() {
+                @Override
+                public void onResponse(AsyncSearchResponse response) {
+                    sendFinalResponse(request, response, nowInMillis, listener);
+                }
+
+                @Override
+                public void onFailure(Exception exc) {
+                    listener.onFailure(exc);
+                }
+            }, request.getWaitForCompletion());
+        } catch (Exception exc) {
+            listener.onFailure(exc);
+        }
+    }
+
+   private void getSearchResponseFromIndex(AsyncSearchId searchId,
+                                           GetAsyncSearchAction.Request request,
+                                           long nowInMillis,
+                                           ActionListener<AsyncSearchResponse> listener) {
+        store.getResponse(searchId, new ActionListener<>() {
+                @Override
+                public void onResponse(AsyncSearchResponse response) {
+                    sendFinalResponse(request, response, nowInMillis, listener);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e);
+                }
+            });
+    }
+
+    private void sendFinalResponse(GetAsyncSearchAction.Request request,
+                                   AsyncSearchResponse response,
+                                   long nowInMillis,
+                                   ActionListener<AsyncSearchResponse> listener) {
+        // check if the result has expired
+        if (response.getExpirationTime() < nowInMillis) {
+            listener.onFailure(new ResourceNotFoundException(request.getId()));
+            return;
+        }
+
+        listener.onResponse(response);
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
new file mode 100644
index 0000000000000..a89f166866a9c
--- /dev/null
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
@@ -0,0 +1,201 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.engine.DocumentMissingException;
+import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
+
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class TransportSubmitAsyncSearchAction extends HandledTransportAction<SubmitAsyncSearchRequest, AsyncSearchResponse> {
+    private static final Logger logger = LogManager.getLogger(TransportSubmitAsyncSearchAction.class);
+
+    private final NodeClient nodeClient;
+    private final Supplier<ReduceContext> reduceContextSupplier;
+    private final TransportSearchAction searchAction;
+    private final AsyncSearchIndexService store;
+
+    @Inject
+    public TransportSubmitAsyncSearchAction(ClusterService clusterService,
+                                            TransportService transportService,
+                                            ActionFilters actionFilters,
+                                            NamedWriteableRegistry registry,
+                                            Client client,
+                                            NodeClient nodeClient,
+                                            SearchService searchService,
+                                            TransportSearchAction searchAction) {
+        super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new);
+        this.nodeClient = nodeClient;
+        this.reduceContextSupplier = () -> searchService.createReduceContext(true);
+        this.searchAction = searchAction;
+        this.store = new AsyncSearchIndexService(clusterService, transportService.getThreadPool().getThreadContext(), client, registry);
+    }
+
+    @Override
+    protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener<AsyncSearchResponse> submitListener) {
+        CancellableTask submitTask = (CancellableTask) task;
+        final SearchRequest searchRequest = createSearchRequest(request, submitTask.getId(), request.getKeepAlive());
+        AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest);
+        searchAction.execute(searchTask, searchRequest, searchTask.getSearchProgressActionListener());
+        searchTask.addCompletionListener(
+            new ActionListener<>() {
+                @Override
+                public void onResponse(AsyncSearchResponse searchResponse) {
+                    if (searchResponse.isRunning() || request.isCleanOnCompletion() == false) {
+                        // the task is still running and the user cannot wait more so we create
+                        // a document for further retrieval
+                        try {
+                            if (submitTask.isCancelled()) {
+                                // the user cancelled the submit so we don't store anything
+                                // and propagate the failure
+                                Exception cause = new TaskCancelledException(submitTask.getReasonCancelled());
+                                onFatalFailure(searchTask, cause, false, submitListener);
+                            } else {
+                                final String docId = searchTask.getSearchId().getDocId();
+                                // creates the fallback response if the node crashes/restarts in the middle of the request
+                                // TODO: store intermediate results ?
+                                AsyncSearchResponse initialResp = searchResponse.clone(searchResponse.getId());
+                                store.storeInitialResponse(docId, searchTask.getOriginHeaders(), initialResp,
+                                    new ActionListener<>() {
+                                        @Override
+                                        public void onResponse(IndexResponse r) {
+                                            if (searchResponse.isRunning()) {
+                                                try {
+                                                    // store the final response on completion unless the submit is cancelled
+                                                    searchTask.addCompletionListener(finalResponse ->
+                                                        onFinalResponse(submitTask, searchTask, finalResponse, () -> {}));
+                                                } finally {
+                                                    submitListener.onResponse(searchResponse);
+                                                }
+                                            } else {
+                                                onFinalResponse(submitTask, searchTask, searchResponse,
+                                                    () -> submitListener.onResponse(searchResponse));
+                                            }
+                                        }
+
+                                        @Override
+                                        public void onFailure(Exception exc) {
+                                            onFatalFailure(searchTask, exc, searchResponse.isRunning(), submitListener);
+                                        }
+                                    });
+                            }
+                        } catch (Exception exc) {
+                            onFatalFailure(searchTask, exc, searchResponse.isRunning(), submitListener);
+                        }
+                    } else {
+                        // the task completed within the timeout so the response is sent back to the user
+                        // with a null id since nothing was stored on the cluster.
+                        taskManager.unregister(searchTask);
+                        submitListener.onResponse(searchResponse.clone(null));
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception exc) {
+                    submitListener.onFailure(exc);
+                }
+            }, request.getWaitForCompletion());
+    }
+
+    private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, long parentTaskId, TimeValue keepAlive) {
+        String docID = UUIDs.randomBase64UUID();
+        Map<String, String> originHeaders = nodeClient.threadPool().getThreadContext().getHeaders();
+        SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) {
+            @Override
+            public AsyncSearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> taskHeaders) {
+                AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id));
+                return new AsyncSearchTask(id, type, action, parentTaskId, keepAlive, originHeaders, taskHeaders, searchId,
+                    store.getClient(), nodeClient.threadPool(), reduceContextSupplier);
+            }
+        };
+        searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), parentTaskId));
+        return searchRequest;
+    }
+
+    private void onFatalFailure(AsyncSearchTask task, Exception error, boolean shouldCancel, ActionListener<AsyncSearchResponse> listener) {
+        if (shouldCancel) {
+            task.cancelTask(() -> {
+                try {
+                    task.addCompletionListener(finalResponse -> taskManager.unregister(task));
+                } finally {
+                    listener.onFailure(error);
+                }
+            });
+        } else {
+            try {
+                task.addCompletionListener(finalResponse -> taskManager.unregister(task));
+            } finally {
+                listener.onFailure(error);
+            }
+        }
+    }
+
+    private void onFinalResponse(CancellableTask submitTask,
+                                 AsyncSearchTask searchTask,
+                                 AsyncSearchResponse response,
+                                 Runnable nextAction) {
+        if (submitTask.isCancelled() || searchTask.isCancelled()) {
+            // the user cancelled the submit so we ensure that there is nothing stored in the response index.
+            store.deleteResponse(searchTask.getSearchId(), false, ActionListener.wrap(() -> {
+                taskManager.unregister(searchTask);
+                nextAction.run();
+            }));
+            return;
+        }
+
+        try {
+            store.storeFinalResponse(searchTask.getSearchId().getDocId(), response, new ActionListener<>() {
+                @Override
+                public void onResponse(UpdateResponse updateResponse) {
+                    taskManager.unregister(searchTask);
+                    nextAction.run();
+                }
+
+                @Override
+                public void onFailure(Exception exc) {
+                    if (exc.getCause() instanceof DocumentMissingException == false) {
+                        logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]",
+                            searchTask.getSearchId().getEncoded()), exc);
+                    }
+                    taskManager.unregister(searchTask);
+                    nextAction.run();
+                }
+            });
+        } catch (Exception exc) {
+            logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", searchTask.getSearchId().getEncoded()), exc);
+            taskManager.unregister(searchTask);
+            nextAction.run();
+        }
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
new file mode 100644
index 0000000000000..373030fe12efa
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.metrics.InternalMax;
+import org.elasticsearch.search.aggregations.metrics.InternalMin;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+// TODO: add tests for keepAlive and expiration
+public class AsyncSearchActionTests extends AsyncSearchIntegTestCase {
+    private String indexName;
+    private int numShards;
+
+    private int numKeywords;
+    private Map<String, AtomicInteger> keywordFreqs;
+    private float maxMetric = Float.NEGATIVE_INFINITY;
+    private float minMetric = Float.POSITIVE_INFINITY;
+
+    @Before
+    public void indexDocuments() throws InterruptedException {
+        indexName = "test-async";
+        numShards = randomIntBetween(internalCluster().numDataNodes(), internalCluster().numDataNodes()*10);
+        int numDocs = randomIntBetween(numShards, numShards*3);
+        createIndex(indexName, Settings.builder().put("index.number_of_shards", numShards).build());
+        numKeywords = randomIntBetween(1, 100);
+        keywordFreqs = new HashMap<>();
+        Set<String> keywordSet = new HashSet<>();
+        for (int i = 0; i < numKeywords; i++) {
+            keywordSet.add(randomAlphaOfLengthBetween(10, 20));
+        }
+        numKeywords = keywordSet.size();
+        String[] keywords = keywordSet.toArray(String[]::new);
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            float metric = randomFloat();
+            maxMetric = Math.max(metric, maxMetric);
+            minMetric = Math.min(metric, minMetric);
+            String keyword = keywords[randomIntBetween(0, numKeywords-1)];
+            keywordFreqs.compute(keyword,
+                (k, v) -> {
+                    if (v == null) {
+                        return new AtomicInteger(1);
+                    }
+                    v.incrementAndGet();
+                    return v;
+            });
+            reqs.add(client().prepareIndex(indexName).setSource("terms", keyword, "metric", metric));
+        }
+        indexRandom(true, true, reqs);
+        ensureGreen("test-async");
+    }
+
+    public void testMaxMinAggregation() throws Exception {
+        int step = numShards > 2 ? randomIntBetween(2, numShards) : 2;
+        int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0;
+        SearchSourceBuilder source = new SearchSourceBuilder()
+            .aggregation(AggregationBuilders.min("min").field("metric"))
+            .aggregation(AggregationBuilders.max("max").field("metric"));
+        try (SearchResponseIterator it =
+                 assertBlockingIterator(indexName, source, numFailures, step)) {
+            AsyncSearchResponse response = it.next();
+            while (it.hasNext()) {
+                response = it.next();
+                assertNotNull(response.getSearchResponse());
+                if (response.getSearchResponse().getSuccessfulShards() > 0) {
+                    assertNotNull(response.getSearchResponse().getAggregations());
+                    assertNotNull(response.getSearchResponse().getAggregations().get("max"));
+                    assertNotNull(response.getSearchResponse().getAggregations().get("min"));
+                    InternalMax max = response.getSearchResponse().getAggregations().get("max");
+                    InternalMin min = response.getSearchResponse().getAggregations().get("min");
+                    assertThat((float) min.getValue(), greaterThanOrEqualTo(minMetric));
+                    assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric));
+                }
+            }
+            if (numFailures == numShards) {
+                assertNotNull(response.getFailure());
+            } else {
+                assertNotNull(response.getSearchResponse());
+                assertNotNull(response.getSearchResponse().getAggregations());
+                assertNotNull(response.getSearchResponse().getAggregations().get("max"));
+                assertNotNull(response.getSearchResponse().getAggregations().get("min"));
+                InternalMax max = response.getSearchResponse().getAggregations().get("max");
+                InternalMin min = response.getSearchResponse().getAggregations().get("min");
+                if (numFailures == 0) {
+                    assertThat((float) min.getValue(), equalTo(minMetric));
+                    assertThat((float) max.getValue(), equalTo(maxMetric));
+                } else {
+                    assertThat((float) min.getValue(), greaterThanOrEqualTo(minMetric));
+                    assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric));
+                }
+            }
+            deleteAsyncSearch(response.getId());
+            ensureTaskRemoval(response.getId());
+        }
+    }
+
+    public void testTermsAggregation() throws Exception {
+        int step = numShards > 2 ? randomIntBetween(2, numShards) : 2;
+        int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0;
+        SearchSourceBuilder source = new SearchSourceBuilder()
+            .aggregation(AggregationBuilders.terms("terms").field("terms.keyword").size(numKeywords));
+        try (SearchResponseIterator it =
+                 assertBlockingIterator(indexName, source, numFailures, step)) {
+            AsyncSearchResponse response = it.next();
+            while (it.hasNext()) {
+                response = it.next();
+                assertNotNull(response.getSearchResponse());
+                if (response.getSearchResponse().getSuccessfulShards() > 0) {
+                    assertNotNull(response.getSearchResponse().getAggregations());
+                    assertNotNull(response.getSearchResponse().getAggregations().get("terms"));
+                    StringTerms terms = response.getSearchResponse().getAggregations().get("terms");
+                    assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0));
+                    assertThat(terms.getBuckets().size(), lessThanOrEqualTo(numKeywords));
+                    for (InternalTerms.Bucket<?> bucket : terms.getBuckets()) {
+                        long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get();
+                        assertThat(bucket.getDocCount(), lessThanOrEqualTo(count));
+                    }
+                }
+            }
+            if (numFailures == numShards) {
+                assertNotNull(response.getFailure());
+            } else {
+                assertNotNull(response.getSearchResponse());
+                assertNotNull(response.getSearchResponse().getAggregations());
+                assertNotNull(response.getSearchResponse().getAggregations().get("terms"));
+                StringTerms terms = response.getSearchResponse().getAggregations().get("terms");
+                assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0));
+                assertThat(terms.getBuckets().size(), lessThanOrEqualTo(numKeywords));
+                for (InternalTerms.Bucket<?> bucket : terms.getBuckets()) {
+                    long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get();
+                    if (numFailures > 0) {
+                        assertThat(bucket.getDocCount(), lessThanOrEqualTo(count));
+                    } else {
+                        assertThat(bucket.getDocCount(), equalTo(count));
+                    }
+                }
+            }
+            deleteAsyncSearch(response.getId());
+            ensureTaskRemoval(response.getId());
+        }
+    }
+
+    public void testRestartAfterCompletion() throws Exception {
+        final AsyncSearchResponse initial;
+        try (SearchResponseIterator it =
+                 assertBlockingIterator(indexName, new SearchSourceBuilder(), 0, 2)) {
+            initial = it.next();
+        }
+        ensureTaskCompletion(initial.getId());
+        restartTaskNode(initial.getId());
+        AsyncSearchResponse response = getAsyncSearch(initial.getId());
+        assertNotNull(response.getSearchResponse());
+        assertFalse(response.isRunning());
+        assertFalse(response.isPartial());
+        deleteAsyncSearch(response.getId());
+        ensureTaskRemoval(response.getId());
+    }
+
+    public void testDeleteCancelRunningTask() throws Exception {
+        final AsyncSearchResponse initial;
+        SearchResponseIterator it =
+            assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2);
+        initial = it.next();
+        deleteAsyncSearch(initial.getId());
+        it.close();
+        ensureTaskCompletion(initial.getId());
+        ensureTaskRemoval(initial.getId());
+    }
+
+    public void testDeleteCleanupIndex() throws Exception {
+        SearchResponseIterator it =
+            assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2);
+        AsyncSearchResponse response = it.next();
+        deleteAsyncSearch(response.getId());
+        it.close();
+        ensureTaskCompletion(response.getId());
+        ensureTaskRemoval(response.getId());
+    }
+
+    public void testCleanupOnFailure() throws Exception {
+        final AsyncSearchResponse initial;
+        try (SearchResponseIterator it =
+                 assertBlockingIterator(indexName, new SearchSourceBuilder(), numShards, 2)) {
+            initial = it.next();
+        }
+        ensureTaskCompletion(initial.getId());
+        AsyncSearchResponse response = getAsyncSearch(initial.getId());
+        assertFalse(response.isRunning());
+        assertNotNull(response.getFailure());
+        assertTrue(response.isPartial());
+        assertThat(response.getSearchResponse().getTotalShards(), equalTo(numShards));
+        assertThat(response.getSearchResponse().getShardFailures().length, equalTo(numShards));
+        deleteAsyncSearch(initial.getId());
+        ensureTaskRemoval(initial.getId());
+    }
+
+    public void testInvalidId() throws Exception {
+        SearchResponseIterator it =
+            assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2);
+        AsyncSearchResponse response = it.next();
+        ExecutionException exc = expectThrows(ExecutionException.class, () -> getAsyncSearch("invalid"));
+        assertThat(exc.getCause(), instanceOf(IllegalArgumentException.class));
+        assertThat(exc.getMessage(), containsString("invalid id"));
+        while (it.hasNext()) {
+            response = it.next();
+        }
+        assertFalse(response.isRunning());
+    }
+
+    public void testNoIndex() throws Exception {
+        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest("invalid-*");
+        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
+        AsyncSearchResponse response = submitAsyncSearch(request);
+        assertNotNull(response.getSearchResponse());
+        assertFalse(response.isRunning());
+        assertThat(response.getSearchResponse().getTotalShards(), equalTo(0));
+
+        request = new SubmitAsyncSearchRequest("invalid");
+        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
+        response = submitAsyncSearch(request);
+        assertNull(response.getSearchResponse());
+        assertNotNull(response.getFailure());
+        assertFalse(response.isRunning());
+        ElasticsearchException exc = response.getFailure();
+        assertThat(exc.getMessage(), containsString("no such index"));
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java
new file mode 100644
index 0000000000000..ebfa4bfc97190
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.ESTestCase;
+
+public class AsyncSearchIdTests extends ESTestCase {
+    public void testEncode() {
+        for (int i = 0; i < 10; i++) {
+            AsyncSearchId instance = new AsyncSearchId(UUIDs.randomBase64UUID(),
+                new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong()));
+            String encoded = AsyncSearchId.encode(instance.getDocId(), instance.getTaskId());
+            AsyncSearchId same = AsyncSearchId.decode(encoded);
+            assertEquals(same, instance);
+
+            AsyncSearchId mutate = mutate(instance);
+            assertNotEquals(mutate, instance);
+            assertNotEquals(mutate, same);
+        }
+    }
+
+    private AsyncSearchId mutate(AsyncSearchId id) {
+        int rand = randomIntBetween(0, 1);
+        switch (rand) {
+            case 0:
+                return new AsyncSearchId(randomAlphaOfLength(id.getDocId().length()+1), id.getTaskId());
+
+            case 1:
+                return new AsyncSearchId(id.getDocId(),
+                    new TaskId(randomAlphaOfLength(id.getTaskId().getNodeId().length()), randomNonNegativeLong()));
+
+            default:
+                throw new AssertionError();
+        }
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
new file mode 100644
index 0000000000000..afc7627b2e4df
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses;
+import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse;
+import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse;
+import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId;
+
+// TODO: test CRUD operations
+public class AsyncSearchIndexServiceTests extends ESSingleNodeTestCase {
+    private AsyncSearchIndexService indexService;
+
+    @Before
+    public void setup() {
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        TransportService transportService = getInstanceFromNode(TransportService.class);
+        indexService = new AsyncSearchIndexService(clusterService, transportService.getThreadPool().getThreadContext(),
+            client(), writableRegistry());
+    }
+
+    public void testEncodeSearchResponse() throws IOException {
+        for (int i = 0; i < 10; i++) {
+            AsyncSearchResponse response = randomAsyncSearchResponse(randomSearchId(), randomSearchResponse());
+            String encoded = indexService.encodeResponse(response);
+            AsyncSearchResponse same = indexService.decodeResponse(encoded);
+            assertEqualResponses(response, same);
+        }
+    }
+
+    public void testEnsuredAuthenticatedUserIsSame() throws IOException {
+        Authentication original =
+            new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null);
+        Authentication current = randomBoolean() ? original :
+            new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null);
+        assertTrue(indexService.ensureAuthenticatedUserIsSame(original, current));
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        original.writeToContext(threadContext);
+        assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current));
+
+        // original is not set
+        assertTrue(indexService.ensureAuthenticatedUserIsSame(Collections.emptyMap(), current));
+        // current is not set
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), null));
+
+        // original user being run as
+        User user = new User(new User("test", "role"), new User("authenticated", "runas"));
+        current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"),
+            new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
+        assertTrue(indexService.ensureAuthenticatedUserIsSame(original, current));
+        assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current));
+
+        // both user are run as
+        current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"),
+            new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
+        Authentication runAs = current;
+        assertTrue(indexService.ensureAuthenticatedUserIsSame(runAs, current));
+        threadContext = new ThreadContext(Settings.EMPTY);
+        original.writeToContext(threadContext);
+        assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current));
+
+        // different authenticated by type
+        Authentication differentRealmType =
+            new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", randomAlphaOfLength(5), "node"), null);
+        threadContext = new ThreadContext(Settings.EMPTY);
+        original.writeToContext(threadContext);
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, differentRealmType));
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), differentRealmType));
+
+        // wrong user
+        Authentication differentUser =
+            new Authentication(new User("test2", "role"), new Authentication.RealmRef("realm", "realm", "node"), null);
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, differentUser));
+
+        // run as different user
+        Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")),
+            new Authentication.RealmRef("realm", "file", "node1"), new Authentication.RealmRef("realm", "file", "node1"));
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, diffRunAs));
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), diffRunAs));
+
+        // run as different looked up by type
+        Authentication runAsDiffType = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"),
+            new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node"));
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(original, runAsDiffType));
+        assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), runAsDiffType));
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
new file mode 100644
index 0000000000000..e5e65a832db8e
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
@@ -0,0 +1,394 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.reindex.ReindexPlugin;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.search.AsyncSearchIndexService.INDEX;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public abstract class AsyncSearchIntegTestCase extends ESIntegTestCase {
+    interface SearchResponseIterator extends Iterator<AsyncSearchResponse>, Closeable {}
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class,
+            QueryBlockPlugin.class, ReindexPlugin.class);
+    }
+
+    /**
+     * Restart the node that runs the {@link TaskId} decoded from the provided {@link AsyncSearchId}.
+     */
+    protected void restartTaskNode(String id) throws Exception {
+        AsyncSearchId searchId = AsyncSearchId.decode(id);
+        final ClusterStateResponse clusterState = client().admin().cluster()
+            .prepareState().clear().setNodes(true).get();
+        DiscoveryNode node = clusterState.getState().nodes().get(searchId.getTaskId().getNodeId());
+        internalCluster().restartNode(node.getName(), new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                return super.onNodeStopped(nodeName);
+            }
+        });
+        ensureYellow(INDEX);
+    }
+
+    protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request) throws ExecutionException, InterruptedException {
+        return client().execute(SubmitAsyncSearchAction.INSTANCE, request).get();
+    }
+
+    protected AsyncSearchResponse getAsyncSearch(String id) throws ExecutionException, InterruptedException {
+        return client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncSearchAction.Request(id)).get();
+    }
+
+    protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionException, InterruptedException {
+        return client().execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(id)).get();
+    }
+
+    /**
+     * Wait the removal of the document decoded from the provided {@link AsyncSearchId}.
+     */
+    protected void ensureTaskRemoval(String id) throws Exception {
+        AsyncSearchId searchId = AsyncSearchId.decode(id);
+        assertBusy(() -> {
+            GetResponse resp = client().prepareGet()
+                .setIndex(INDEX)
+                .setId(searchId.getDocId())
+                .get();
+            assertFalse(resp.isExists());
+        });
+    }
+
+    /**
+     * Wait the completion of the {@link TaskId} decoded from the provided {@link AsyncSearchId}.
+     */
+    protected void ensureTaskCompletion(String id) throws Exception {
+        assertBusy(() -> {
+            TaskId taskId = AsyncSearchId.decode(id).getTaskId();
+            try {
+                GetTaskResponse resp = client().admin().cluster()
+                    .prepareGetTask(taskId).get();
+                assertNull(resp.getTask());
+            } catch (Exception exc) {
+                if (exc.getCause() instanceof ResourceNotFoundException == false) {
+                    throw exc;
+                }
+            }
+        });
+    }
+
+    protected SearchResponseIterator assertBlockingIterator(String indexName,
+                                                            SearchSourceBuilder source,
+                                                            int numFailures,
+                                                            int progressStep) throws Exception {
+        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(source, indexName);
+        request.setBatchedReduceSize(progressStep);
+        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
+        ClusterSearchShardsResponse response = dataNodeClient().admin().cluster()
+            .prepareSearchShards(request.getSearchRequest().indices()).get();
+        AtomicInteger failures = new AtomicInteger(numFailures);
+        Map<ShardId, ShardIdLatch> shardLatchMap = Arrays.stream(response.getGroups())
+            .map(ClusterSearchShardsGroup::getShardId)
+            .collect(
+                Collectors.toMap(
+                    Function.identity(),
+                    id -> new ShardIdLatch(id, new CountDownLatch(1), failures.decrementAndGet() >= 0)
+                )
+            );
+        ShardIdLatch[] shardLatchArray = shardLatchMap.values().stream()
+            .sorted(Comparator.comparing(ShardIdLatch::shard))
+            .toArray(ShardIdLatch[]::new);
+        resetPluginsLatch(shardLatchMap);
+        request.getSearchRequest().source().query(new BlockQueryBuilder(shardLatchMap));
+
+        final AsyncSearchResponse initial = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get();
+
+        assertTrue(initial.isPartial());
+        assertThat(initial.status(), equalTo(RestStatus.OK));
+        assertThat(initial.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
+        assertThat(initial.getSearchResponse().getSuccessfulShards(), equalTo(0));
+        assertThat(initial.getSearchResponse().getShardFailures().length, equalTo(0));
+
+        return new SearchResponseIterator() {
+            private AsyncSearchResponse response = initial;
+            private int shardIndex = 0;
+            private boolean isFirst = true;
+
+            @Override
+            public boolean hasNext() {
+                return response.isRunning();
+            }
+
+            @Override
+            public AsyncSearchResponse next() {
+                try {
+                    return doNext();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            private AsyncSearchResponse doNext() throws Exception {
+                if (isFirst) {
+                    isFirst = false;
+                    return response;
+                }
+                AtomicReference<AsyncSearchResponse> atomic = new AtomicReference<>();
+                int step = shardIndex == 0 ? progressStep+1 : progressStep-1;
+                int index = 0;
+                while (index < step && shardIndex < shardLatchArray.length) {
+                    if (shardLatchArray[shardIndex].shouldFail == false) {
+                        ++index;
+                    }
+                    shardLatchArray[shardIndex++].countDown();
+                }
+                AsyncSearchResponse newResponse = client().execute(GetAsyncSearchAction.INSTANCE,
+                    new GetAsyncSearchAction.Request(response.getId())
+                        .setWaitForCompletion(TimeValue.timeValueMillis(10))).get();
+
+                if (newResponse.isRunning()) {
+                    assertThat(newResponse.status(),  equalTo(RestStatus.OK));
+                    assertTrue(newResponse.isPartial());
+                    assertNull(newResponse.getFailure());
+                    assertNotNull(newResponse.getSearchResponse());
+                    assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
+                    assertThat(newResponse.getSearchResponse().getShardFailures().length, lessThanOrEqualTo(numFailures));
+                } else if (numFailures == shardLatchArray.length) {
+                    assertThat(newResponse.status(),  equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+                    assertNotNull(newResponse.getFailure());
+                    assertTrue(newResponse.isPartial());
+                    assertNotNull(newResponse.getSearchResponse());
+                    assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
+                    assertThat(newResponse.getSearchResponse().getSuccessfulShards(), equalTo(0));
+                    assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures));
+                    assertNull(newResponse.getSearchResponse().getAggregations());
+                    assertNotNull(newResponse.getSearchResponse().getHits().getTotalHits());
+                    assertThat(newResponse.getSearchResponse().getHits().getTotalHits().value, equalTo(0L));
+                    assertThat(newResponse.getSearchResponse().getHits().getTotalHits().relation,
+                        equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+                } else {
+                    assertThat(newResponse.status(),  equalTo(RestStatus.OK));
+                    assertNotNull(newResponse.getSearchResponse());
+                    assertFalse(newResponse.isPartial());
+                    assertThat(newResponse.status(), equalTo(RestStatus.OK));
+                    assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length));
+                    assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures));
+                    assertThat(newResponse.getSearchResponse().getSuccessfulShards(),
+                        equalTo(shardLatchArray.length-newResponse.getSearchResponse().getShardFailures().length));
+                }
+                return response = newResponse;
+            }
+
+            @Override
+            public void close() {
+                Arrays.stream(shardLatchArray).forEach(shard -> {
+                    if (shard.latch.getCount() == 1) {
+                        shard.latch.countDown();
+                    }
+                });
+            }
+        };
+    }
+
+    private void resetPluginsLatch(Map<ShardId, ShardIdLatch> newLatch) {
+        for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) {
+            pluginsService.filterPlugins(QueryBlockPlugin.class).forEach(p -> p.reset(newLatch));
+        }
+    }
+
+    public static class QueryBlockPlugin extends Plugin implements SearchPlugin {
+        private Map<ShardId, ShardIdLatch> shardsLatch;
+
+        public QueryBlockPlugin() {
+            this.shardsLatch = null;
+        }
+
+        public void reset(Map<ShardId, ShardIdLatch> newLatch) {
+            shardsLatch = newLatch;
+        }
+
+        @Override
+        public List<QuerySpec<?>> getQueries() {
+            return Collections.singletonList(
+                new QuerySpec<>("block_match_all",
+                    in -> new BlockQueryBuilder(in, shardsLatch),
+                    p -> BlockQueryBuilder.fromXContent(p, shardsLatch))
+            );
+        }
+    }
+
+    private static class BlockQueryBuilder extends AbstractQueryBuilder<BlockQueryBuilder> {
+        public static final String NAME = "block_match_all";
+        private final Map<ShardId, ShardIdLatch> shardsLatch;
+
+        private BlockQueryBuilder(Map<ShardId, ShardIdLatch> shardsLatch) {
+            super();
+            this.shardsLatch = shardsLatch;
+        }
+
+        BlockQueryBuilder(StreamInput in, Map<ShardId, ShardIdLatch> shardsLatch) throws IOException {
+            super(in);
+            this.shardsLatch = shardsLatch;
+        }
+
+        private BlockQueryBuilder() {
+            this.shardsLatch = null;
+        }
+
+        @Override
+        protected void doWriteTo(StreamOutput out) {}
+
+        @Override
+        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(NAME);
+            builder.endObject();
+        }
+
+        private static final ObjectParser<BlockQueryBuilder, Void> PARSER = new ObjectParser<>(NAME, BlockQueryBuilder::new);
+
+        public static BlockQueryBuilder fromXContent(XContentParser parser, Map<ShardId, ShardIdLatch> shardsLatch) {
+            try {
+                PARSER.apply(parser, null);
+                return new BlockQueryBuilder(shardsLatch);
+            } catch (IllegalArgumentException e) {
+                throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
+            }
+        }
+
+        @Override
+        protected Query doToQuery(QueryShardContext context) {
+            final Query delegate = Queries.newMatchAllQuery();
+            return new Query() {
+                @Override
+                public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+                    if (shardsLatch != null) {
+                        try {
+                            final ShardIdLatch latch = shardsLatch.get(new ShardId(context.index(), context.getShardId()));
+                            latch.await();
+                            if (latch.shouldFail) {
+                                throw new IOException("boum");
+                            }
+                        } catch (InterruptedException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                    return delegate.createWeight(searcher, scoreMode, boost);
+                }
+
+                @Override
+                public String toString(String field) {
+                    return delegate.toString(field);
+                }
+
+                @Override
+                public boolean equals(Object obj) {
+                    return false;
+                }
+
+                @Override
+                public int hashCode() {
+                    return 0;
+                }
+            };
+        }
+
+        @Override
+        protected boolean doEquals(BlockQueryBuilder other) {
+            return false;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return 0;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+    }
+
+    private static class ShardIdLatch {
+        private final ShardId shard;
+        private final CountDownLatch latch;
+        private final boolean shouldFail;
+
+        private ShardIdLatch(ShardId shard, CountDownLatch latch, boolean shouldFail) {
+            this.shard = shard;
+            this.latch = latch;
+            this.shouldFail = shouldFail;
+        }
+
+        ShardId shard() {
+            return shard;
+        }
+
+        void countDown() {
+            latch.countDown();
+        }
+
+        void await() throws InterruptedException {
+            latch.await();
+        }
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
new file mode 100644
index 0000000000000..925ce9d7897eb
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider;
+import org.elasticsearch.xpack.core.transform.transforms.SyncConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId;
+
+public class AsyncSearchResponseTests extends ESTestCase {
+    private SearchResponse searchResponse = randomSearchResponse();
+    private NamedWriteableRegistry namedWriteableRegistry;
+
+    @Before
+    public void registerNamedObjects() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
+
+        List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(),
+            TimeSyncConfig::new));
+
+        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers());
+
+        namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
+    }
+
+
+    protected Writeable.Reader<AsyncSearchResponse> instanceReader() {
+        return AsyncSearchResponse::new;
+    }
+
+    protected AsyncSearchResponse createTestInstance() {
+        return randomAsyncSearchResponse(randomSearchId(), searchResponse);
+    }
+
+    protected void assertEqualInstances(AsyncSearchResponse expectedInstance, AsyncSearchResponse newInstance) {
+        assertNotSame(newInstance, expectedInstance);
+        assertEqualResponses(expectedInstance, newInstance);
+    }
+
+    public final void testSerialization() throws IOException {
+        for (int runs = 0; runs < 10; runs++) {
+            AsyncSearchResponse testInstance = createTestInstance();
+            assertSerialization(testInstance);
+        }
+    }
+
+    protected final AsyncSearchResponse assertSerialization(AsyncSearchResponse testInstance) throws IOException {
+        return assertSerialization(testInstance, Version.CURRENT);
+    }
+
+    protected final AsyncSearchResponse assertSerialization(AsyncSearchResponse testInstance, Version version) throws IOException {
+        AsyncSearchResponse deserializedInstance = copyInstance(testInstance, version);
+        assertEqualInstances(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+    protected final AsyncSearchResponse copyInstance(AsyncSearchResponse instance) throws IOException {
+        return copyInstance(instance, Version.CURRENT);
+    }
+
+    protected AsyncSearchResponse copyInstance(AsyncSearchResponse instance, Version version) throws IOException {
+        return copyWriteable(instance, namedWriteableRegistry, instanceReader(), version);
+    }
+
+    static AsyncSearchResponse randomAsyncSearchResponse(String searchId, SearchResponse searchResponse) {
+        int rand = randomIntBetween(0, 2);
+        switch (rand) {
+            case 0:
+                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean(),
+                    randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong());
+
+            case 1:
+                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse, null,
+                    randomBoolean(), randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong());
+
+            case 2:
+                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse,
+                    new ElasticsearchException(new IOException("boum")), randomBoolean(), randomBoolean(),
+                    randomNonNegativeLong(), randomNonNegativeLong());
+
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    static SearchResponse randomSearchResponse() {
+        long tookInMillis = randomNonNegativeLong();
+        int totalShards = randomIntBetween(1, Integer.MAX_VALUE);
+        int successfulShards = randomIntBetween(0, totalShards);
+        int skippedShards = totalShards - successfulShards;
+        InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
+        return new SearchResponse(internalSearchResponse, null, totalShards,
+            successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+    }
+
+    static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchResponse actual) {
+        assertEquals(expected.getId(), actual.getId());
+        assertEquals(expected.getVersion(), actual.getVersion());
+        assertEquals(expected.status(), actual.status());
+        assertEquals(expected.getFailure() == null, actual.getFailure() == null);
+        assertEquals(expected.isRunning(), actual.isRunning());
+        assertEquals(expected.isPartial(), actual.isPartial());
+        assertEquals(expected.getStartTime(), actual.getStartTime());
+        assertEquals(expected.getExpirationTime(), actual.getExpirationTime());
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
new file mode 100644
index 0000000000000..8c3f57883ec14
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
@@ -0,0 +1,182 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchShard;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AsyncSearchTaskTests extends ESTestCase {
+    private ThreadPool threadPool;
+
+    @Before
+    public void beforeTest() {
+        threadPool = new TestThreadPool(getTestName());
+    }
+
+    @After
+    public void afterTest() {
+        threadPool.shutdownNow();
+    }
+
+    public void testWaitForInit() throws InterruptedException {
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+            Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
+            new NoOpClient(threadPool), threadPool, null);
+        int numShards = randomIntBetween(0, 10);
+        List<SearchShard> shards = new ArrayList<>();
+        for (int i = 0; i < numShards; i++) {
+            shards.add(new SearchShard(null, new ShardId("0", "0", 1)));
+        }
+        List<SearchShard> skippedShards = new ArrayList<>();
+        int numSkippedShards = randomIntBetween(0, 10);
+        for (int i = 0; i < numSkippedShards; i++) {
+            skippedShards.add(new SearchShard(null, new ShardId("0", "0", 1)));
+        }
+
+        int numThreads = randomIntBetween(1, 10);
+        CountDownLatch latch = new CountDownLatch(numThreads);
+        for (int i = 0; i < numThreads; i++) {
+            Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() {
+                @Override
+                public void onResponse(AsyncSearchResponse resp) {
+                    assertThat(numShards + numSkippedShards, equalTo(resp.getSearchResponse().getTotalShards()));
+                    assertThat(numSkippedShards, equalTo(resp.getSearchResponse().getSkippedShards()));
+                    assertThat(0, equalTo(resp.getSearchResponse().getFailedShards()));
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    throw new AssertionError(e);
+
+                }
+            }, TimeValue.timeValueMillis(1)));
+            thread.start();
+        }
+        assertFalse(latch.await(numThreads*2, TimeUnit.MILLISECONDS));
+        task.getSearchProgressActionListener().onListShards(shards, skippedShards, SearchResponse.Clusters.EMPTY, false);
+        latch.await();
+    }
+
+    public void testWithFailure() throws InterruptedException {
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+            Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
+            new NoOpClient(threadPool), threadPool, null);
+        int numThreads = randomIntBetween(1, 10);
+        CountDownLatch latch = new CountDownLatch(numThreads);
+        for (int i = 0; i < numThreads; i++) {
+            Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() {
+                @Override
+                public void onResponse(AsyncSearchResponse resp) {
+                    assertNull(resp.getSearchResponse());
+                    assertNotNull(resp.getFailure());
+                    assertTrue(resp.isPartial());
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    throw new AssertionError(e);
+                }
+            }, TimeValue.timeValueMillis(1)));
+            thread.start();
+        }
+        assertFalse(latch.await(numThreads*2, TimeUnit.MILLISECONDS));
+        task.getSearchProgressActionListener().onFailure(new Exception("boom"));
+        latch.await();
+    }
+
+    public void testWaitForCompletion() throws InterruptedException {
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+            Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
+            new NoOpClient(threadPool), threadPool, null);
+        int numShards = randomIntBetween(0, 10);
+        List<SearchShard> shards = new ArrayList<>();
+        for (int i = 0; i < numShards; i++) {
+            shards.add(new SearchShard(null, new ShardId("0", "0", 1)));
+        }
+        List<SearchShard> skippedShards = new ArrayList<>();
+        int numSkippedShards = randomIntBetween(0, 10);
+        for (int i = 0; i < numSkippedShards; i++) {
+            skippedShards.add(new SearchShard(null, new ShardId("0", "0", 1)));
+        }
+
+        int numShardFailures = 0;
+        task.getSearchProgressActionListener().onListShards(shards, skippedShards, SearchResponse.Clusters.EMPTY, false);
+        for (int i = 0; i < numShards; i++) {
+            task.getSearchProgressActionListener().onPartialReduce(shards.subList(i, i+1),
+                new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), null, 0);
+            assertCompletionListeners(task, numShards+numSkippedShards, numSkippedShards, numShardFailures, true);
+        }
+        task.getSearchProgressActionListener().onFinalReduce(shards,
+            new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), null, 0);
+        assertCompletionListeners(task, numShards+numSkippedShards, numSkippedShards, numShardFailures, true);
+        ((AsyncSearchTask.Listener)task.getProgressListener()).onResponse(
+            newSearchResponse(numShards+numSkippedShards, numShards, numSkippedShards));
+        assertCompletionListeners(task, numShards+numSkippedShards,
+            numSkippedShards, numShardFailures, false);
+        threadPool.shutdownNow();
+    }
+
+    private static SearchResponse newSearchResponse(int totalShards, int successfulShards, int skippedShards) {
+        InternalSearchResponse response = new InternalSearchResponse(SearchHits.empty(),
+            InternalAggregations.EMPTY, null, null, false, null, 1);
+        return new SearchResponse(response, null, totalShards, successfulShards, skippedShards,
+            100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+    }
+
+    private void assertCompletionListeners(AsyncSearchTask task,
+                                           int expectedTotalShards,
+                                           int expectedSkippedShards,
+                                           int expectedShardFailures,
+                                           boolean isPartial) throws InterruptedException {
+        int numThreads = randomIntBetween(1, 10);
+        CountDownLatch latch = new CountDownLatch(numThreads);
+        for (int i = 0; i < numThreads; i++) {
+            Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() {
+                @Override
+                public void onResponse(AsyncSearchResponse resp) {
+                    assertThat(resp.getSearchResponse().getTotalShards(), equalTo(expectedTotalShards));
+                    assertThat(resp.getSearchResponse().getSkippedShards(), equalTo(expectedSkippedShards));
+                    assertThat(resp.getSearchResponse().getFailedShards(), equalTo(expectedShardFailures));
+                    assertThat(resp.isPartial(), equalTo(isPartial));
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    throw new AssertionError(e);
+                }
+            }, TimeValue.timeValueMillis(1)));
+            thread.start();
+        }
+        latch.await();
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
new file mode 100644
index 0000000000000..f71d859f648a3
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+
+import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId;
+
+public class DeleteAsyncSearchRequestTests extends AbstractWireSerializingTestCase<DeleteAsyncSearchAction.Request> {
+    @Override
+    protected Writeable.Reader<DeleteAsyncSearchAction.Request> instanceReader() {
+        return DeleteAsyncSearchAction.Request::new;
+    }
+
+    @Override
+    protected DeleteAsyncSearchAction.Request createTestInstance() {
+        return new DeleteAsyncSearchAction.Request(randomSearchId());
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java
new file mode 100644
index 0000000000000..5e3ec4ded0ec8
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+
+public class GetAsyncSearchRequestTests extends AbstractWireSerializingTestCase<GetAsyncSearchAction.Request> {
+    @Override
+    protected Writeable.Reader<GetAsyncSearchAction.Request> instanceReader() {
+        return GetAsyncSearchAction.Request::new;
+    }
+
+    @Override
+    protected GetAsyncSearchAction.Request createTestInstance() {
+        GetAsyncSearchAction.Request req = new GetAsyncSearchAction.Request(randomSearchId());
+        if (randomBoolean()) {
+            req.setWaitForCompletion(TimeValue.timeValueMillis(randomIntBetween(1, 10000)));
+        }
+        if (randomBoolean()) {
+            req.setKeepAlive(TimeValue.timeValueMillis(randomIntBetween(1, 10000)));
+        }
+        return req;
+    }
+
+    static String randomSearchId() {
+        return AsyncSearchId.encode(UUIDs.randomBase64UUID(),
+            new TaskId(randomAlphaOfLengthBetween(10, 20), randomLongBetween(0, Long.MAX_VALUE)));
+    }
+
+    public void testValidateWaitForCompletion() {
+
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
new file mode 100644
index 0000000000000..2aae3817205e2
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
+import org.elasticsearch.xpack.core.transform.action.AbstractWireSerializingTransformTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SubmitAsyncSearchRequestTests extends AbstractWireSerializingTransformTestCase<SubmitAsyncSearchRequest> {
+    @Override
+    protected Writeable.Reader<SubmitAsyncSearchRequest> instanceReader() {
+        return SubmitAsyncSearchRequest::new;
+    }
+
+    @Override
+    protected SubmitAsyncSearchRequest createTestInstance() {
+        final SubmitAsyncSearchRequest searchRequest;
+        if (randomBoolean()) {
+            searchRequest = new SubmitAsyncSearchRequest(generateRandomStringArray(10, 10, false, false));
+        }  else {
+            searchRequest = new SubmitAsyncSearchRequest();
+        }
+        if (randomBoolean()) {
+            searchRequest.setWaitForCompletion(TimeValue.parseTimeValue(randomPositiveTimeValue(), "wait_for_completion"));
+        }
+        searchRequest.setCleanOnCompletion(randomBoolean());
+        if (randomBoolean()) {
+            searchRequest.setKeepAlive(TimeValue.parseTimeValue(randomPositiveTimeValue(), "keep_alive"));
+        }
+        if (randomBoolean()) {
+            searchRequest.getSearchRequest()
+                .indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+        }
+        if (randomBoolean()) {
+            searchRequest.getSearchRequest()
+                .preference(randomAlphaOfLengthBetween(3, 10));
+        }
+        if (randomBoolean()) {
+            searchRequest.getSearchRequest().requestCache(randomBoolean());
+        }
+        if (randomBoolean()) {
+            searchRequest.getSearchRequest().searchType(randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH));
+        }
+        if (randomBoolean()) {
+            searchRequest.getSearchRequest().source(randomSearchSourceBuilder());
+        }
+        return searchRequest;
+    }
+
+    protected SearchSourceBuilder randomSearchSourceBuilder() {
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        if (randomBoolean()) {
+            source.query(QueryBuilders.termQuery("foo", "bar"));
+        }
+        if (randomBoolean()) {
+            source.aggregation(AggregationBuilders.max("max").field("field"));
+        }
+        return source;
+    }
+
+    public void testValidateCssMinimizeRoundtrips() {
+        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
+        req.getSearchRequest().setCcsMinimizeRoundtrips(true);
+        ActionRequestValidationException exc = req.validate();
+        assertNotNull(exc);
+        assertThat(exc.validationErrors().size(), equalTo(1));
+        assertThat(exc.validationErrors().get(0), containsString("[ccs_minimize_roundtrips]"));
+    }
+
+    public void testValidateScroll() {
+        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
+        req.getSearchRequest().scroll(TimeValue.timeValueMinutes(5));
+        ActionRequestValidationException exc = req.validate();
+        assertNotNull(exc);
+        assertThat(exc.validationErrors().size(), equalTo(2));
+        // request_cache is activated by default
+        assertThat(exc.validationErrors().get(0), containsString("[request_cache]"));
+        assertThat(exc.validationErrors().get(1), containsString("[scroll]"));
+    }
+
+    public void testValidateKeepAlive() {
+        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
+        req.setKeepAlive(TimeValue.timeValueSeconds(randomIntBetween(1, 59)));
+        ActionRequestValidationException exc = req.validate();
+        assertNotNull(exc);
+        assertThat(exc.validationErrors().size(), equalTo(1));
+        assertThat(exc.validationErrors().get(0), containsString("[keep_alive]"));
+    }
+
+    public void testValidateSuggestOnly() {
+        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
+        req.getSearchRequest().source(new SearchSourceBuilder().suggest(new SuggestBuilder()));
+        ActionRequestValidationException exc = req.validate();
+        assertNotNull(exc);
+        assertThat(exc.validationErrors().size(), equalTo(1));
+        assertThat(exc.validationErrors().get(0), containsString("suggest"));
+    }
+}
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index 85bf1a7500d46..e2867b1d715a8 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -105,6 +105,8 @@ testClusters.integTest {
   keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
   user username: "x_pack_rest_user", password: "x-pack-test-password"
+  user username: "cat_test_user", password: "cat-test-password", role: "cat_test_role"
   extraConfigFile nodeKey.name, nodeKey
   extraConfigFile nodeCert.name, nodeCert
+  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
index d1270aeefd203..28cefdd0a57b4 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
@@ -395,7 +395,7 @@ protected void asyncShardOperation(
 
                         @Override
                         public Executor executor() {
-                            return threadPool.executor(ThreadPool.Names.LISTENER);
+                            return threadPool.executor(Ccr.CCR_THREAD_POOL_NAME);
                         }
 
                         @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
index cbc4e4abc2bfe..e523d452c75b1 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
@@ -132,16 +132,7 @@ protected void innerUpdateMapping(long minRequiredMappingVersion, LongConsumer h
                 final Index followerIndex = params.getFollowShardId().getIndex();
                 final Index leaderIndex = params.getLeaderShardId().getIndex();
                 final Supplier<TimeValue> timeout = () -> isStopped() ? TimeValue.MINUS_ONE : waitForMetadataTimeOut;
-
-                final Client remoteClient;
-                try {
-                    remoteClient = remoteClient(params);
-                } catch (NoSuchRemoteClusterException e) {
-                    errorHandler.accept(e);
-                    return;
-                }
-
-                CcrRequests.getIndexMetadata(remoteClient, leaderIndex, minRequiredMappingVersion, 0L, timeout, ActionListener.wrap(
+                final ActionListener<IndexMetaData> listener = ActionListener.wrap(
                     indexMetaData -> {
                         if (indexMetaData.mapping() == null) {
                             assert indexMetaData.getMappingVersion() == 1;
@@ -155,7 +146,12 @@ protected void innerUpdateMapping(long minRequiredMappingVersion, LongConsumer h
                             errorHandler));
                     },
                     errorHandler
-                ));
+                );
+                try {
+                    CcrRequests.getIndexMetadata(remoteClient(params), leaderIndex, minRequiredMappingVersion, 0L, timeout, listener);
+                } catch (NoSuchRemoteClusterException e) {
+                    errorHandler.accept(e);
+                }
             }
 
             @Override
@@ -424,21 +420,27 @@ protected Scheduler.Cancellable scheduleBackgroundRetentionLeaseRenewal(final Lo
                                         "{} background adding retention lease [{}] while following",
                                         params.getFollowShardId(),
                                         retentionLeaseId);
-                                CcrRetentionLeases.asyncAddRetentionLease(
+                                try {
+                                    final ActionListener<RetentionLeaseActions.Response> wrappedListener = ActionListener.wrap(
+                                        r -> {},
+                                        inner -> {
+                                            /*
+                                             * If this fails that the retention lease already exists, something highly unusual is
+                                             * going on. Log it, and renew again after another renew interval has passed.
+                                             */
+                                            final Throwable innerCause = ExceptionsHelper.unwrapCause(inner);
+                                            logRetentionLeaseFailure(retentionLeaseId, innerCause);
+                                        });
+                                    CcrRetentionLeases.asyncAddRetentionLease(
                                         params.getLeaderShardId(),
                                         retentionLeaseId,
                                         followerGlobalCheckpoint.getAsLong(),
                                         remoteClient(params),
-                                        ActionListener.wrap(
-                                                r -> {},
-                                                inner -> {
-                                                    /*
-                                                     * If this fails that the retention lease already exists, something highly unusual is
-                                                     * going on. Log it, and renew again after another renew interval has passed.
-                                                     */
-                                                    final Throwable innerCause = ExceptionsHelper.unwrapCause(inner);
-                                                    logRetentionLeaseFailure(retentionLeaseId, innerCause);
-                                                }));
+                                        wrappedListener);
+                                } catch (NoSuchRemoteClusterException rce) {
+                                    // we will attempt to renew again after another renew interval has passed
+                                    logRetentionLeaseFailure(retentionLeaseId, rce);
+                                }
                             } else {
                                  // if something else happened, we will attempt to renew again after another renew interval has passed
                             }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index 7181dd9b6c9a2..c349ea764101f 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -225,7 +225,6 @@ private NodeConfigurationSource createNodeConfigurationSource(final String leade
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
-        builder.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         // Let cluster state api return quickly in order to speed up auto follow tests:
         builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100));
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
index cbfd078452368..9858362c811de 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
@@ -48,7 +48,6 @@ protected Settings nodeSettings() {
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
-        builder.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         // Let cluster state api return quickly in order to speed up auto follow tests:
         builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index 1d90ef9488789..4e58023eb260b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -613,6 +613,16 @@ public boolean isFlattenedAllowed() {
     public boolean isVectorsAllowed() {
         return allowForAllLicenses();
     }
+    
+
+    /**
+     * Determine if Wildcard support should be enabled.
+     * <p>
+     *  Wildcard is available for all license types except {@link OperationMode#MISSING}
+     */
+    public synchronized boolean isWildcardAllowed() {
+        return status.active;
+    }    
 
     public boolean isOdbcAllowed() {
         return isAllowedByLicense(OperationMode.PLATINUM);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 16a3ce6d5bcb6..26abe1af66fa0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -34,7 +34,9 @@
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
-import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage;
@@ -420,7 +422,11 @@ public List<ActionType<? extends ActionResponse>> getClientActions() {
                 DeleteTransformAction.INSTANCE,
                 GetTransformAction.INSTANCE,
                 GetTransformStatsAction.INSTANCE,
-                PreviewTransformAction.INSTANCE
+                PreviewTransformAction.INSTANCE,
+                // Async Search
+                SubmitAsyncSearchAction.INSTANCE,
+                GetAsyncSearchAction.INSTANCE,
+                DeleteAsyncSearchAction.INSTANCE
         );
     }
 
@@ -432,8 +438,6 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new),
                 // logstash
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.LOGSTASH, LogstashFeatureSetUsage::new),
-                // beats
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.BEATS, BeatsFeatureSetUsage::new),
                 // ML - Custom metadata
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index 82e9f2553c3ed..740d1015a7eb6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -82,11 +82,7 @@ private XPackSettings() {
 
     /** Setting for enabling or disabling Logstash extensions. Defaults to true. */
     public static final Setting<Boolean> LOGSTASH_ENABLED = Setting.boolSetting("xpack.logstash.enabled", true,
-            Setting.Property.NodeScope);
-
-    /** Setting for enabling or disabling Beats extensions. Defaults to true. */
-    public static final Setting<Boolean> BEATS_ENABLED = Setting.boolSetting("xpack.beats.enabled", true,
-        Setting.Property.NodeScope);
+            Setting.Property.NodeScope, Property.Deprecated);
 
     /**
      * Setting for enabling or disabling the index lifecycle extension. Defaults to true.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java
deleted file mode 100644
index 1702bf3869df6..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.beats;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
-
-import java.io.IOException;
-
-public final class BeatsFeatureSetUsage extends XPackFeatureSet.Usage {
-
-    public BeatsFeatureSetUsage(StreamInput in) throws IOException {
-        super(in);
-    }
-
-    public BeatsFeatureSetUsage(boolean available, boolean enabled) {
-        super(XPackField.BEATS, available, enabled);
-    }
-
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
index 7d621a62cf359..43cf8b603fdd0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
@@ -114,9 +114,8 @@ public AutoFollowStats(StreamInput in) throws IOException {
         numberOfFailedFollowIndices = in.readVLong();
         numberOfFailedRemoteClusterStateRequests = in.readVLong();
         numberOfSuccessfulFollowIndices = in.readVLong();
-        // note: the casts to the following Writeable.Reader<T> instances are needed by some IDEs (e.g. Eclipse 4.8) as a compiler help
-        recentAutoFollowErrors = new TreeMap<>(in.readMap(StreamInput::readString,
-            in1 -> new Tuple<>(in1.readZLong(), in1.readException())));
+        recentAutoFollowErrors = new TreeMap<>(
+                in.readMap(StreamInput::readString, in1 -> new Tuple<>(in1.readZLong(), in1.readException())));
         autoFollowedClusters = new TreeMap<>(in.readMap(StreamInput::readString, AutoFollowedCluster::new));
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
index 3b67e5bdc17d3..fa5f4c13491d6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
@@ -26,7 +27,12 @@ public FreezeStep(StepKey key, StepKey nextStepKey, Client client) {
     public void performDuringNoSnapshot(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
         getClient().admin().indices().execute(FreezeIndexAction.INSTANCE,
             new FreezeRequest(indexMetaData.getIndex().getName()).masterNodeTimeout(getMasterTimeout(currentState)),
-            ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure));
+            ActionListener.wrap(response -> {
+                if (response.isAcknowledged() == false) {
+                    throw new ElasticsearchException("freeze index request failed to be acknowledged");
+                }
+                listener.onResponse(true);
+            }, listener::onFailure));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java
new file mode 100644
index 0000000000000..c1839af5a6ec0
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class EstimateModelMemoryAction extends ActionType<EstimateModelMemoryAction.Response> {
+
+    public static final EstimateModelMemoryAction INSTANCE = new EstimateModelMemoryAction();
+    public static final String NAME = "cluster:admin/xpack/ml/job/estimate_model_memory";
+
+    private EstimateModelMemoryAction() {
+        super(NAME, Response::new);
+    }
+
+    public static class Request extends ActionRequest {
+
+        public static final ParseField ANALYSIS_CONFIG = Job.ANALYSIS_CONFIG;
+        public static final ParseField OVERALL_CARDINALITY = new ParseField("overall_cardinality");
+        public static final ParseField MAX_BUCKET_CARDINALITY = new ParseField("max_bucket_cardinality");
+
+        public static final ObjectParser<Request, Void> PARSER =
+            new ObjectParser<>(NAME, EstimateModelMemoryAction.Request::new);
+
+        static {
+            PARSER.declareObject(Request::setAnalysisConfig, (p, c) -> AnalysisConfig.STRICT_PARSER.apply(p, c).build(), ANALYSIS_CONFIG);
+            PARSER.declareObject(Request::setOverallCardinality,
+                (p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, OVERALL_CARDINALITY)),
+                OVERALL_CARDINALITY);
+            PARSER.declareObject(Request::setMaxBucketCardinality,
+                (p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, MAX_BUCKET_CARDINALITY)),
+                MAX_BUCKET_CARDINALITY);
+        }
+
+        public static Request parseRequest(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        private AnalysisConfig analysisConfig;
+        private Map<String, Long> overallCardinality = Collections.emptyMap();
+        private Map<String, Long> maxBucketCardinality = Collections.emptyMap();
+
+        public Request() {
+            super();
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.analysisConfig = in.readBoolean() ? new AnalysisConfig(in) : null;
+            this.overallCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
+            this.maxBucketCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            if (analysisConfig != null) {
+                out.writeBoolean(true);
+                analysisConfig.writeTo(out);
+            } else {
+                out.writeBoolean(false);
+            }
+            out.writeMap(overallCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
+            out.writeMap(maxBucketCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            if (analysisConfig == null) {
+                ActionRequestValidationException e = new ActionRequestValidationException();
+                e.addValidationError("[" + ANALYSIS_CONFIG.getPreferredName() + "] was not specified");
+                return e;
+            }
+            return null;
+        }
+
+        public AnalysisConfig getAnalysisConfig() {
+            return analysisConfig;
+        }
+
+        public void setAnalysisConfig(AnalysisConfig analysisConfig) {
+            this.analysisConfig = ExceptionsHelper.requireNonNull(analysisConfig, ANALYSIS_CONFIG);
+        }
+
+        public Map<String, Long> getOverallCardinality() {
+            return overallCardinality;
+        }
+
+        public void setOverallCardinality(Map<String, Long> overallCardinality) {
+            this.overallCardinality =
+                Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(overallCardinality, OVERALL_CARDINALITY));
+        }
+
+        public Map<String, Long> getMaxBucketCardinality() {
+            return maxBucketCardinality;
+        }
+
+        public void setMaxBucketCardinality(Map<String, Long> maxBucketCardinality) {
+            this.maxBucketCardinality =
+                Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(maxBucketCardinality, MAX_BUCKET_CARDINALITY));
+        }
+
+        private static long parseNonNegativeLong(XContentParser parser, ParseField enclosingField) throws IOException {
+            long value = parser.longValue();
+            if (value < 0) {
+                throw ExceptionsHelper.badRequestException("[{}] contained negative cardinality [{}]",
+                    enclosingField.getPreferredName(), value);
+            }
+            return value;
+        }
+    }
+
+    public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
+
+        public RequestBuilder(ElasticsearchClient client, EstimateModelMemoryAction action) {
+            super(client, action, new Request());
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private static final ParseField MODEL_MEMORY_ESTIMATE = new ParseField("model_memory_estimate");
+
+        private final ByteSizeValue modelMemoryEstimate;
+
+        public Response(ByteSizeValue modelMemoryEstimate) {
+            this.modelMemoryEstimate = Objects.requireNonNull(modelMemoryEstimate);
+        }
+
+        public Response(StreamInput in) throws IOException {
+            modelMemoryEstimate = new ByteSizeValue(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            modelMemoryEstimate.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(MODEL_MEMORY_ESTIMATE.getPreferredName(), modelMemoryEstimate.getStringRep());
+            builder.endObject();
+            return builder;
+        }
+
+        public ByteSizeValue getModelMemoryEstimate() {
+            return modelMemoryEstimate;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java
index 04ced8a2f4c06..664162bb80cb4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParams.java
@@ -33,7 +33,7 @@ public class BoostedTreeParams implements ToXContentFragment, Writeable {
     public static final ParseField LAMBDA = new ParseField("lambda");
     public static final ParseField GAMMA = new ParseField("gamma");
     public static final ParseField ETA = new ParseField("eta");
-    public static final ParseField MAXIMUM_NUMBER_TREES = new ParseField("maximum_number_trees");
+    public static final ParseField MAX_TREES = new ParseField("max_trees", "maximum_number_trees");
     public static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
     public static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
 
@@ -41,7 +41,7 @@ static void declareFields(AbstractObjectParser<?, Void> parser) {
         parser.declareDouble(optionalConstructorArg(), LAMBDA);
         parser.declareDouble(optionalConstructorArg(), GAMMA);
         parser.declareDouble(optionalConstructorArg(), ETA);
-        parser.declareInt(optionalConstructorArg(), MAXIMUM_NUMBER_TREES);
+        parser.declareInt(optionalConstructorArg(), MAX_TREES);
         parser.declareDouble(optionalConstructorArg(), FEATURE_BAG_FRACTION);
         parser.declareInt(optionalConstructorArg(), NUM_TOP_FEATURE_IMPORTANCE_VALUES);
     }
@@ -49,14 +49,14 @@ static void declareFields(AbstractObjectParser<?, Void> parser) {
     private final Double lambda;
     private final Double gamma;
     private final Double eta;
-    private final Integer maximumNumberTrees;
+    private final Integer maxTrees;
     private final Double featureBagFraction;
     private final Integer numTopFeatureImportanceValues;
 
     public BoostedTreeParams(@Nullable Double lambda,
                              @Nullable Double gamma,
                              @Nullable Double eta,
-                             @Nullable Integer maximumNumberTrees,
+                             @Nullable Integer maxTrees,
                              @Nullable Double featureBagFraction,
                              @Nullable Integer numTopFeatureImportanceValues) {
         if (lambda != null && lambda < 0) {
@@ -68,8 +68,8 @@ public BoostedTreeParams(@Nullable Double lambda,
         if (eta != null && (eta < 0.001 || eta > 1)) {
             throw ExceptionsHelper.badRequestException("[{}] must be a double in [0.001, 1]", ETA.getPreferredName());
         }
-        if (maximumNumberTrees != null && (maximumNumberTrees <= 0 || maximumNumberTrees > 2000)) {
-            throw ExceptionsHelper.badRequestException("[{}] must be an integer in [1, 2000]", MAXIMUM_NUMBER_TREES.getPreferredName());
+        if (maxTrees != null && (maxTrees <= 0 || maxTrees > 2000)) {
+            throw ExceptionsHelper.badRequestException("[{}] must be an integer in [1, 2000]", MAX_TREES.getPreferredName());
         }
         if (featureBagFraction != null && (featureBagFraction <= 0 || featureBagFraction > 1.0)) {
             throw ExceptionsHelper.badRequestException("[{}] must be a double in (0, 1]", FEATURE_BAG_FRACTION.getPreferredName());
@@ -81,7 +81,7 @@ public BoostedTreeParams(@Nullable Double lambda,
         this.lambda = lambda;
         this.gamma = gamma;
         this.eta = eta;
-        this.maximumNumberTrees = maximumNumberTrees;
+        this.maxTrees = maxTrees;
         this.featureBagFraction = featureBagFraction;
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
     }
@@ -90,7 +90,7 @@ public BoostedTreeParams(@Nullable Double lambda,
         lambda = in.readOptionalDouble();
         gamma = in.readOptionalDouble();
         eta = in.readOptionalDouble();
-        maximumNumberTrees = in.readOptionalVInt();
+        maxTrees = in.readOptionalVInt();
         featureBagFraction = in.readOptionalDouble();
         if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
             numTopFeatureImportanceValues = in.readOptionalInt();
@@ -104,7 +104,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalDouble(lambda);
         out.writeOptionalDouble(gamma);
         out.writeOptionalDouble(eta);
-        out.writeOptionalVInt(maximumNumberTrees);
+        out.writeOptionalVInt(maxTrees);
         out.writeOptionalDouble(featureBagFraction);
         if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
             out.writeOptionalInt(numTopFeatureImportanceValues);
@@ -122,8 +122,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (eta != null) {
             builder.field(ETA.getPreferredName(), eta);
         }
-        if (maximumNumberTrees != null) {
-            builder.field(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
+        if (maxTrees != null) {
+            builder.field(MAX_TREES.getPreferredName(), maxTrees);
         }
         if (featureBagFraction != null) {
             builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -145,8 +145,8 @@ Map<String, Object> getParams() {
         if (eta != null) {
             params.put(ETA.getPreferredName(), eta);
         }
-        if (maximumNumberTrees != null) {
-            params.put(MAXIMUM_NUMBER_TREES.getPreferredName(), maximumNumberTrees);
+        if (maxTrees != null) {
+            params.put(MAX_TREES.getPreferredName(), maxTrees);
         }
         if (featureBagFraction != null) {
             params.put(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
@@ -165,14 +165,14 @@ public boolean equals(Object o) {
         return Objects.equals(lambda, that.lambda)
             && Objects.equals(gamma, that.gamma)
             && Objects.equals(eta, that.eta)
-            && Objects.equals(maximumNumberTrees, that.maximumNumberTrees)
+            && Objects.equals(maxTrees, that.maxTrees)
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(numTopFeatureImportanceValues, that.numTopFeatureImportanceValues);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues);
+        return Objects.hash(lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues);
     }
 
     public static Builder builder() {
@@ -184,7 +184,7 @@ public static class Builder {
         private Double lambda;
         private Double gamma;
         private Double eta;
-        private Integer maximumNumberTrees;
+        private Integer maxTrees;
         private Double featureBagFraction;
         private Integer numTopFeatureImportanceValues;
 
@@ -194,7 +194,7 @@ private Builder() {}
             this.lambda = params.lambda;
             this.gamma = params.gamma;
             this.eta = params.eta;
-            this.maximumNumberTrees = params.maximumNumberTrees;
+            this.maxTrees = params.maxTrees;
             this.featureBagFraction = params.featureBagFraction;
             this.numTopFeatureImportanceValues = params.numTopFeatureImportanceValues;
         }
@@ -214,8 +214,8 @@ public Builder setEta(Double eta) {
             return this;
         }
 
-        public Builder setMaximumNumberTrees(Integer maximumNumberTrees) {
-            this.maximumNumberTrees = maximumNumberTrees;
+        public Builder setMaxTrees(Integer maxTrees) {
+            this.maxTrees = maxTrees;
             return this;
         }
 
@@ -230,7 +230,7 @@ public Builder setNumTopFeatureImportanceValues(Integer numTopFeatureImportanceV
         }
 
         public BoostedTreeParams build() {
-            return new BoostedTreeParams(lambda, gamma, eta, maximumNumberTrees, featureBagFraction, numTopFeatureImportanceValues);
+            return new BoostedTreeParams(lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues);
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
index d9289a5bcf1ac..c0965f8d7ebb5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.FieldAliasMapper;
@@ -21,6 +22,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -37,15 +39,23 @@ public class Classification implements DataFrameAnalysis {
 
     public static final ParseField DEPENDENT_VARIABLE = new ParseField("dependent_variable");
     public static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
+    public static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
     public static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
     public static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
     public static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
     private static final String STATE_DOC_ID_SUFFIX = "_classification_state#1";
 
+    private static final String NUM_CLASSES = "num_classes";
+
     private static final ConstructingObjectParser<Classification, Void> LENIENT_PARSER = createParser(true);
     private static final ConstructingObjectParser<Classification, Void> STRICT_PARSER = createParser(false);
 
+    /**
+     * The max number of classes classification supports
+     */
+    private static final int MAX_DEPENDENT_VARIABLE_CARDINALITY = 30;
+
     private static ConstructingObjectParser<Classification, Void> createParser(boolean lenient) {
         ConstructingObjectParser<Classification, Void> parser = new ConstructingObjectParser<>(
             NAME.getPreferredName(),
@@ -54,12 +64,19 @@ private static ConstructingObjectParser<Classification, Void> createParser(boole
                 (String) a[0],
                 new BoostedTreeParams((Double) a[1], (Double) a[2], (Double) a[3], (Integer) a[4], (Double) a[5], (Integer) a[6]),
                 (String) a[7],
-                (Integer) a[8],
-                (Double) a[9],
-                (Long) a[10]));
+                (ClassAssignmentObjective) a[8],
+                (Integer) a[9],
+                (Double) a[10],
+                (Long) a[11]));
         parser.declareString(constructorArg(), DEPENDENT_VARIABLE);
         BoostedTreeParams.declareFields(parser);
         parser.declareString(optionalConstructorArg(), PREDICTION_FIELD_NAME);
+        parser.declareField(optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ClassAssignmentObjective.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
         parser.declareInt(optionalConstructorArg(), NUM_TOP_CLASSES);
         parser.declareDouble(optionalConstructorArg(), TRAINING_PERCENT);
         parser.declareLong(optionalConstructorArg(), RANDOMIZE_SEED);
@@ -89,6 +106,7 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
     private final String dependentVariable;
     private final BoostedTreeParams boostedTreeParams;
     private final String predictionFieldName;
+    private final ClassAssignmentObjective classAssignmentObjective;
     private final int numTopClasses;
     private final double trainingPercent;
     private final long randomizeSeed;
@@ -96,6 +114,7 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
     public Classification(String dependentVariable,
                           BoostedTreeParams boostedTreeParams,
                           @Nullable String predictionFieldName,
+                          @Nullable ClassAssignmentObjective classAssignmentObjective,
                           @Nullable Integer numTopClasses,
                           @Nullable Double trainingPercent,
                           @Nullable Long randomizeSeed) {
@@ -108,19 +127,26 @@ public Classification(String dependentVariable,
         this.dependentVariable = ExceptionsHelper.requireNonNull(dependentVariable, DEPENDENT_VARIABLE);
         this.boostedTreeParams = ExceptionsHelper.requireNonNull(boostedTreeParams, BoostedTreeParams.NAME);
         this.predictionFieldName = predictionFieldName == null ? dependentVariable + "_prediction" : predictionFieldName;
+        this.classAssignmentObjective = classAssignmentObjective == null ?
+            ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL : classAssignmentObjective;
         this.numTopClasses = numTopClasses == null ? DEFAULT_NUM_TOP_CLASSES : numTopClasses;
         this.trainingPercent = trainingPercent == null ? 100.0 : trainingPercent;
         this.randomizeSeed = randomizeSeed == null ? Randomness.get().nextLong() : randomizeSeed;
     }
 
     public Classification(String dependentVariable) {
-        this(dependentVariable, BoostedTreeParams.builder().build(), null, null, null, null);
+        this(dependentVariable, BoostedTreeParams.builder().build(), null, null, null, null, null);
     }
 
     public Classification(StreamInput in) throws IOException {
         dependentVariable = in.readString();
         boostedTreeParams = new BoostedTreeParams(in);
         predictionFieldName = in.readOptionalString();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            classAssignmentObjective = in.readEnum(ClassAssignmentObjective.class);
+        } else {
+            classAssignmentObjective = ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL;
+        }
         numTopClasses = in.readOptionalVInt();
         trainingPercent = in.readDouble();
         if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
@@ -142,6 +168,10 @@ public String getPredictionFieldName() {
         return predictionFieldName;
     }
 
+    public ClassAssignmentObjective getClassAssignmentObjective() {
+        return classAssignmentObjective;
+    }
+
     public int getNumTopClasses() {
         return numTopClasses;
     }
@@ -164,6 +194,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(dependentVariable);
         boostedTreeParams.writeTo(out);
         out.writeOptionalString(predictionFieldName);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeEnum(classAssignmentObjective);
+        }
         out.writeOptionalVInt(numTopClasses);
         out.writeDouble(trainingPercent);
         if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
@@ -178,6 +211,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         boostedTreeParams.toXContent(builder, params);
+        builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
         builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         if (predictionFieldName != null) {
             builder.field(PREDICTION_FIELD_NAME.getPreferredName(), predictionFieldName);
@@ -191,18 +225,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
+    public Map<String, Object> getParams(FieldInfo fieldInfo) {
         Map<String, Object> params = new HashMap<>();
         params.put(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         params.putAll(boostedTreeParams.getParams());
+        params.put(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
         params.put(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         if (predictionFieldName != null) {
             params.put(PREDICTION_FIELD_NAME.getPreferredName(), predictionFieldName);
         }
-        String predictionFieldType = getPredictionFieldType(extractedFields.get(dependentVariable));
+        String predictionFieldType = getPredictionFieldType(fieldInfo.getTypes(dependentVariable));
         if (predictionFieldType != null) {
             params.put(PREDICTION_FIELD_TYPE, predictionFieldType);
         }
+        params.put(NUM_CLASSES, fieldInfo.getCardinality(dependentVariable));
         return params;
     }
 
@@ -244,7 +280,7 @@ public List<RequiredField> getRequiredFields() {
     @Override
     public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
         // This restriction is due to the fact that currently the C++ backend only supports binomial classification.
-        return Collections.singletonList(FieldCardinalityConstraint.between(dependentVariable, 2, 2));
+        return Collections.singletonList(FieldCardinalityConstraint.between(dependentVariable, 2, MAX_DEPENDENT_VARIABLE_CARDINALITY));
     }
 
     @SuppressWarnings("unchecked")
@@ -303,6 +339,7 @@ public boolean equals(Object o) {
         return Objects.equals(dependentVariable, that.dependentVariable)
             && Objects.equals(boostedTreeParams, that.boostedTreeParams)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
+            && Objects.equals(classAssignmentObjective, that.classAssignmentObjective)
             && Objects.equals(numTopClasses, that.numTopClasses)
             && trainingPercent == that.trainingPercent
             && randomizeSeed == that.randomizeSeed;
@@ -310,6 +347,20 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(dependentVariable, boostedTreeParams, predictionFieldName, numTopClasses, trainingPercent, randomizeSeed);
+        return Objects.hash(dependentVariable, boostedTreeParams, predictionFieldName, classAssignmentObjective,
+                            numTopClasses, trainingPercent, randomizeSeed);
+    }
+
+    public enum ClassAssignmentObjective {
+        MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
+
+        public static ClassAssignmentObjective fromString(String value) {
+            return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
index 664b38e4fc05d..941224dc30a67 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.dataframe.analyses;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 
@@ -16,9 +17,9 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
 
     /**
      * @return The analysis parameters as a map
-     * @param extractedFields map of (name, types) for all the extracted fields
+     * @param fieldInfo Information about the fields like types and cardinalities
      */
-    Map<String, Object> getParams(Map<String, Set<String>> extractedFields);
+    Map<String, Object> getParams(FieldInfo fieldInfo);
 
     /**
      * @return {@code true} if this analysis supports fields with categorical values (i.e. text, keyword, ip)
@@ -64,4 +65,27 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
      * Returns the document id for the analysis state
      */
     String getStateDocId(String jobId);
+
+    /**
+     * Summarizes information about the fields that is necessary for analysis to generate
+     * the parameters needed for the process configuration.
+     */
+    interface FieldInfo {
+
+        /**
+         * Returns the types for the given field or {@code null} if the field is unknown
+         * @param field the field whose types to return
+         * @return the types for the given field or {@code null} if the field is unknown
+         */
+        @Nullable
+        Set<String> getTypes(String field);
+
+        /**
+         * Returns the cardinality of the given field or {@code null} if there is no cardinality for that field
+         * @param field the field whose cardinality to get
+         * @return the cardinality of the given field or {@code null} if there is no cardinality for that field
+         */
+        @Nullable
+        Long getCardinality(String field);
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
index 654d5ba4d1a29..2c83afa87808d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
@@ -192,7 +192,7 @@ public int hashCode() {
     }
 
     @Override
-    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
+    public Map<String, Object> getParams(FieldInfo fieldInfo) {
         Map<String, Object> params = new HashMap<>();
         if (nNeighbors != null) {
             params.put(N_NEIGHBORS.getPreferredName(), nNeighbors);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
index 86f8039090c74..d8c490ddcb804 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
@@ -155,7 +155,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
+    public Map<String, Object> getParams(FieldInfo fieldInfo) {
         Map<String, Object> params = new HashMap<>();
         params.put(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         params.putAll(boostedTreeParams.getParams());
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
index 1a79dff41e10c..9fdba68d4cda7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
@@ -66,7 +66,7 @@ default <T extends EvaluationMetric> List<T> initMetrics(@Nullable List<T> parse
      * Builds the search required to collect data to compute the evaluation result
      * @param userProvidedQueryBuilder User-provided query that must be respected when collecting data
      */
-    default SearchSourceBuilder buildSearch(QueryBuilder userProvidedQueryBuilder) {
+    default SearchSourceBuilder buildSearch(EvaluationParameters parameters, QueryBuilder userProvidedQueryBuilder) {
         Objects.requireNonNull(userProvidedQueryBuilder);
         BoolQueryBuilder boolQuery =
             QueryBuilders.boolQuery()
@@ -78,7 +78,8 @@ default SearchSourceBuilder buildSearch(QueryBuilder userProvidedQueryBuilder) {
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).query(boolQuery);
         for (EvaluationMetric metric : getMetrics()) {
             // Fetch aggregations requested by individual metrics
-            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs = metric.aggs(getActualField(), getPredictedField());
+            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs =
+                metric.aggs(parameters, getActualField(), getPredictedField());
             aggs.v1().forEach(searchSourceBuilder::aggregation);
             aggs.v2().forEach(searchSourceBuilder::aggregation);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java
index 8a106175ace91..a5c3d657f55e9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetric.java
@@ -28,11 +28,14 @@ public interface EvaluationMetric extends ToXContentObject, NamedWriteable {
 
     /**
      * Builds the aggregation that collect required data to compute the metric
+     * @param parameters settings that may be needed by aggregations
      * @param actualField the field that stores the actual value
      * @param predictedField the field that stores the predicted value (class name or probability)
      * @return the aggregations required to compute the metric
      */
-    Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField);
+    Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                           String actualField,
+                                                                           String predictedField);
 
     /**
      * Processes given aggregations as a step towards computing result
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java
new file mode 100644
index 0000000000000..e834efc7e67f7
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParameters.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation;
+
+/**
+ * Encapsulates parameters needed by evaluation.
+ */
+public class EvaluationParameters {
+
+    /**
+     * Maximum number of buckets allowed in any single search request.
+     */
+    private final int maxBuckets;
+
+    public EvaluationParameters(int maxBuckets) {
+        this.maxBuckets = maxBuckets;
+    }
+
+    public int getMaxBuckets() {
+        return maxBuckets;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
index c6636329a65d9..440a46540cecb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
@@ -24,14 +24,13 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -65,12 +64,6 @@ public class Accuracy implements EvaluationMetric {
 
     static final String OVERALL_ACCURACY_AGG_NAME = "classification_overall_accuracy";
 
-    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
-
-    private static Script buildScript(Object...args) {
-        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
-    }
-
     private static final ObjectParser<Accuracy, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Accuracy::new);
 
     public static Accuracy fromXContent(XContentParser parser) {
@@ -103,16 +96,20 @@ public String getName() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                        String actualField,
+                                                                                        String predictedField) {
         // Store given {@code actualField} for the purpose of generating error message in {@code process}.
         this.actualField.trySet(actualField);
         List<AggregationBuilder> aggs = new ArrayList<>();
         List<PipelineAggregationBuilder> pipelineAggs = new ArrayList<>();
         if (overallAccuracy.get() == null) {
-            aggs.add(AggregationBuilders.avg(OVERALL_ACCURACY_AGG_NAME).script(buildScript(actualField, predictedField)));
+            Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
+            aggs.add(AggregationBuilders.avg(OVERALL_ACCURACY_AGG_NAME).script(script));
         }
         if (result.get() == null) {
-            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> matrixAggs = matrix.aggs(actualField, predictedField);
+            Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> matrixAggs =
+                matrix.aggs(parameters, actualField, predictedField);
             aggs.addAll(matrixAggs.v1());
             pipelineAggs.addAll(matrixAggs.v2());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java
index e2762bd7d7986..f041e66698aa6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrix.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.search.aggregations.metrics.Cardinality;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -72,9 +73,9 @@ public static MulticlassConfusionMatrix fromXContent(XContentParser parser) {
     }
 
     static final String STEP_1_AGGREGATE_BY_ACTUAL_CLASS = NAME.getPreferredName() + "_step_1_by_actual_class";
+    static final String STEP_1_CARDINALITY_OF_ACTUAL_CLASS = NAME.getPreferredName() + "_step_1_cardinality_of_actual_class";
     static final String STEP_2_AGGREGATE_BY_ACTUAL_CLASS = NAME.getPreferredName() + "_step_2_by_actual_class";
     static final String STEP_2_AGGREGATE_BY_PREDICTED_CLASS = NAME.getPreferredName() + "_step_2_by_predicted_class";
-    static final String STEP_2_CARDINALITY_OF_ACTUAL_CLASS = NAME.getPreferredName() + "_step_2_cardinality_of_actual_class";
     private static final String OTHER_BUCKET_KEY = "_other_";
     private static final String DEFAULT_AGG_NAME_PREFIX = "";
     private static final int DEFAULT_SIZE = 10;
@@ -83,6 +84,9 @@ public static MulticlassConfusionMatrix fromXContent(XContentParser parser) {
     private final int size;
     private final String aggNamePrefix;
     private final SetOnce<List<String>> topActualClassNames = new SetOnce<>();
+    private final SetOnce<Long> actualClassesCardinality = new SetOnce<>();
+    /** Accumulates actual classes processed so far. It may take more than 1 call to #process method to fill this field completely. */
+    private final List<ActualClass> actualClasses = new ArrayList<>();
     private final SetOnce<Result> result = new SetOnce<>();
 
     public MulticlassConfusionMatrix() {
@@ -121,34 +125,45 @@ public int getSize() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
-        if (topActualClassNames.get() == null) {  // This is step 1
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                        String actualField,
+                                                                                        String predictedField) {
+        if (topActualClassNames.get() == null && actualClassesCardinality.get() == null) {  // This is step 1
             return Tuple.tuple(
                 List.of(
                     AggregationBuilders.terms(aggName(STEP_1_AGGREGATE_BY_ACTUAL_CLASS))
                         .field(actualField)
                         .order(List.of(BucketOrder.count(false), BucketOrder.key(true)))
-                        .size(size)),
+                        .size(size),
+                    AggregationBuilders.cardinality(aggName(STEP_1_CARDINALITY_OF_ACTUAL_CLASS))
+                        .field(actualField)),
                 List.of());
         }
-        if (result.get() == null) {  // This is step 2
-            KeyedFilter[] keyedFiltersActual =
-                topActualClassNames.get().stream()
-                    .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(actualField, className)))
-                    .toArray(KeyedFilter[]::new);
+        if (result.get() == null) {  // These are steps 2, 3, 4 etc.
             KeyedFilter[] keyedFiltersPredicted =
                 topActualClassNames.get().stream()
                     .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(predictedField, className)))
                     .toArray(KeyedFilter[]::new);
-            return Tuple.tuple(
-                List.of(
-                    AggregationBuilders.cardinality(aggName(STEP_2_CARDINALITY_OF_ACTUAL_CLASS))
-                        .field(actualField),
-                    AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS), keyedFiltersActual)
-                        .subAggregation(AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_PREDICTED_CLASS), keyedFiltersPredicted)
-                            .otherBucket(true)
-                            .otherBucketKey(OTHER_BUCKET_KEY))),
-                List.of());
+            // Knowing exactly how many buckets does each aggregation use, we can choose the size of the batch so that
+            // too_many_buckets_exception exception is not thrown.
+            // The only exception is when "search.max_buckets" is set far too low to even have 1 actual class in the batch.
+            // In such case, the exception will be thrown telling the user they should increase the value of "search.max_buckets".
+            int actualClassesPerBatch = Math.max(parameters.getMaxBuckets() / (topActualClassNames.get().size() + 2), 1);
+            KeyedFilter[] keyedFiltersActual =
+                topActualClassNames.get().stream()
+                    .skip(actualClasses.size())
+                    .limit(actualClassesPerBatch)
+                    .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(actualField, className)))
+                    .toArray(KeyedFilter[]::new);
+            if (keyedFiltersActual.length > 0) {
+                return Tuple.tuple(
+                    List.of(
+                        AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS), keyedFiltersActual)
+                            .subAggregation(AggregationBuilders.filters(aggName(STEP_2_AGGREGATE_BY_PREDICTED_CLASS), keyedFiltersPredicted)
+                                .otherBucket(true)
+                                .otherBucketKey(OTHER_BUCKET_KEY))),
+                    List.of());
+            }
         }
         return Tuple.tuple(List.of(), List.of());
     }
@@ -159,10 +174,12 @@ public void process(Aggregations aggs) {
             Terms termsAgg = aggs.get(aggName(STEP_1_AGGREGATE_BY_ACTUAL_CLASS));
             topActualClassNames.set(termsAgg.getBuckets().stream().map(Terms.Bucket::getKeyAsString).sorted().collect(Collectors.toList()));
         }
+        if (actualClassesCardinality.get() == null && aggs.get(aggName(STEP_1_CARDINALITY_OF_ACTUAL_CLASS)) != null) {
+            Cardinality cardinalityAgg = aggs.get(aggName(STEP_1_CARDINALITY_OF_ACTUAL_CLASS));
+            actualClassesCardinality.set(cardinalityAgg.getValue());
+        }
         if (result.get() == null && aggs.get(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS)) != null) {
-            Cardinality cardinalityAgg = aggs.get(aggName(STEP_2_CARDINALITY_OF_ACTUAL_CLASS));
             Filters filtersAgg = aggs.get(aggName(STEP_2_AGGREGATE_BY_ACTUAL_CLASS));
-            List<ActualClass> actualClasses = new ArrayList<>(filtersAgg.getBuckets().size());
             for (Filters.Bucket bucket : filtersAgg.getBuckets()) {
                 String actualClass = bucket.getKeyAsString();
                 long actualClassDocCount = bucket.getDocCount();
@@ -181,7 +198,9 @@ public void process(Aggregations aggs) {
                 predictedClasses.sort(comparing(PredictedClass::getPredictedClass));
                 actualClasses.add(new ActualClass(actualClass, actualClassDocCount, predictedClasses, otherPredictedClassDocCount));
             }
-            result.set(new Result(actualClasses, Math.max(cardinalityAgg.getValue() - size, 0)));
+            if (actualClasses.size() == topActualClassNames.get().size()) {
+                result.set(new Result(actualClasses, Math.max(actualClassesCardinality.get() - size, 0)));
+            }
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
new file mode 100644
index 0000000000000..152bef11c09a2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification;
+
+import org.elasticsearch.script.Script;
+
+import java.text.MessageFormat;
+import java.util.Locale;
+
+/**
+ * Painless scripts used by classification metrics in this package.
+ */
+final class PainlessScripts {
+
+    /**
+     * Template for the comparison script.
+     * It uses "String.valueOf" method in case the mapping types of the two fields are different.
+     */
+    private static final MessageFormat COMPARISON_SCRIPT_TEMPLATE =
+        new MessageFormat("String.valueOf(doc[''{0}''].value).equals(String.valueOf(doc[''{1}''].value))", Locale.ROOT);
+
+    /**
+     * Builds script that tests field values equality for the given actual and predicted field names.
+     *
+     * @param actualField name of the actual field
+     * @param predictedField name of the predicted field
+     * @return script that tests whether the values of actualField and predictedField are equal
+     */
+    static Script buildIsEqualScript(String actualField, String predictedField) {
+        return new Script(COMPARISON_SCRIPT_TEMPLATE.format(new Object[]{ actualField, predictedField }));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
index 73c7723c86b96..c2fe3e4069fcc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
@@ -30,14 +30,13 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -58,17 +57,12 @@ public class Precision implements EvaluationMetric {
 
     public static final ParseField NAME = new ParseField("precision");
 
-    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
     private static final String AGG_NAME_PREFIX = "classification_precision_";
     static final String ACTUAL_CLASSES_NAMES_AGG_NAME = AGG_NAME_PREFIX + "by_actual_class";
     static final String BY_PREDICTED_CLASS_AGG_NAME = AGG_NAME_PREFIX + "by_predicted_class";
     static final String PER_PREDICTED_CLASS_PRECISION_AGG_NAME = AGG_NAME_PREFIX + "per_predicted_class_precision";
     static final String AVG_PRECISION_AGG_NAME = AGG_NAME_PREFIX + "avg_precision";
 
-    private static Script buildScript(Object...args) {
-        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
-    }
-
     private static final ObjectParser<Precision, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Precision::new);
 
     public static Precision fromXContent(XContentParser parser) {
@@ -96,7 +90,9 @@ public String getName() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                        String actualField,
+                                                                                        String predictedField) {
         // Store given {@code actualField} for the purpose of generating error message in {@code process}.
         this.actualField.trySet(actualField);
         if (topActualClassNames.get() == null) {  // This is step 1
@@ -113,7 +109,7 @@ public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> a
                 topActualClassNames.get().stream()
                     .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(predictedField, className)))
                     .toArray(KeyedFilter[]::new);
-            Script script = buildScript(actualField, predictedField);
+            Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
             return Tuple.tuple(
                 List.of(
                     AggregationBuilders.filters(BY_PREDICTED_CLASS_AGG_NAME, keyedFiltersPredicted)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
index 0358820cc509c..2fdd979fcb825 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
@@ -20,20 +20,20 @@
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -53,16 +53,11 @@ public class Recall implements EvaluationMetric {
 
     public static final ParseField NAME = new ParseField("recall");
 
-    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
     private static final String AGG_NAME_PREFIX = "classification_recall_";
     static final String BY_ACTUAL_CLASS_AGG_NAME = AGG_NAME_PREFIX + "by_actual_class";
     static final String PER_ACTUAL_CLASS_RECALL_AGG_NAME = AGG_NAME_PREFIX + "per_actual_class_recall";
     static final String AVG_RECALL_AGG_NAME = AGG_NAME_PREFIX + "avg_recall";
 
-    private static Script buildScript(Object...args) {
-        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
-    }
-
     private static final ObjectParser<Recall, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Recall::new);
 
     public static Recall fromXContent(XContentParser parser) {
@@ -89,17 +84,20 @@ public String getName() {
     }
 
     @Override
-    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
+    public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                        String actualField,
+                                                                                        String predictedField) {
         // Store given {@code actualField} for the purpose of generating error message in {@code process}.
         this.actualField.trySet(actualField);
         if (result.get() != null) {
             return Tuple.tuple(List.of(), List.of());
         }
-        Script script = buildScript(actualField, predictedField);
+        Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
         return Tuple.tuple(
             List.of(
                 AggregationBuilders.terms(BY_ACTUAL_CLASS_AGG_NAME)
                     .field(actualField)
+                    .order(List.of(BucketOrder.count(false), BucketOrder.key(true)))
                     .size(MAX_CLASSES_CARDINALITY)
                     .subAggregation(AggregationBuilders.avg(PER_ACTUAL_CLASS_RECALL_AGG_NAME).script(script))),
             List.of(
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java
index 15a0522d72432..982c68dfb58af 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredError.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.text.MessageFormat;
@@ -65,7 +66,9 @@ public String getName() {
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                  String actualField,
+                                                                                  String predictedField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java
index d3a04c415c047..ac142631ab968 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.text.MessageFormat;
@@ -70,7 +71,9 @@ public String getName() {
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                  String actualField,
+                                                                                  String predictedField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
index e09e03b1208aa..fdd7cd2ce2d6c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -65,7 +66,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedProbabilityField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                  String actualField,
+                                                                                  String predictedProbabilityField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
index 3f62a0b144eec..db0f9b95d1380 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.search.aggregations.metrics.Percentiles;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -127,7 +128,9 @@ public int hashCode() {
     }
 
     @Override
-    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedProbabilityField) {
+    public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                  String actualField,
+                                                                                  String predictedProbabilityField) {
         if (result != null) {
             return Tuple.tuple(List.of(), List.of());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
index 33e19cb2882d2..e7cb624016dd8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
@@ -31,6 +31,7 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -60,6 +61,7 @@ public class TrainedModelConfig implements ToXContentObject, Writeable {
     public static final ParseField ESTIMATED_HEAP_MEMORY_USAGE_BYTES = new ParseField("estimated_heap_memory_usage_bytes");
     public static final ParseField ESTIMATED_OPERATIONS = new ParseField("estimated_operations");
     public static final ParseField LICENSE_LEVEL = new ParseField("license_level");
+    public static final ParseField DEFAULT_FIELD_MAP = new ParseField("default_field_map");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
     public static final ObjectParser<TrainedModelConfig.Builder, Void> LENIENT_PARSER = createParser(true);
@@ -90,6 +92,7 @@ private static ObjectParser<TrainedModelConfig.Builder, Void> createParser(boole
             DEFINITION);
         parser.declareString(TrainedModelConfig.Builder::setLazyDefinition, COMPRESSED_DEFINITION);
         parser.declareString(TrainedModelConfig.Builder::setLicenseLevel, LICENSE_LEVEL);
+        parser.declareObject(TrainedModelConfig.Builder::setDefaultFieldMap, (p, c) -> p.mapStrings(), DEFAULT_FIELD_MAP);
         return parser;
     }
 
@@ -108,6 +111,7 @@ public static TrainedModelConfig.Builder fromXContent(XContentParser parser, boo
     private final long estimatedHeapMemory;
     private final long estimatedOperations;
     private final License.OperationMode licenseLevel;
+    private final Map<String, String> defaultFieldMap;
 
     private final LazyModelDefinition definition;
 
@@ -122,7 +126,8 @@ public static TrainedModelConfig.Builder fromXContent(XContentParser parser, boo
                        TrainedModelInput input,
                        Long estimatedHeapMemory,
                        Long estimatedOperations,
-                       String licenseLevel) {
+                       String licenseLevel,
+                       Map<String, String> defaultFieldMap) {
         this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
         this.createdBy = ExceptionsHelper.requireNonNull(createdBy, CREATED_BY);
         this.version = ExceptionsHelper.requireNonNull(version, VERSION);
@@ -142,6 +147,7 @@ public static TrainedModelConfig.Builder fromXContent(XContentParser parser, boo
         }
         this.estimatedOperations = estimatedOperations;
         this.licenseLevel = License.OperationMode.parse(ExceptionsHelper.requireNonNull(licenseLevel, LICENSE_LEVEL));
+        this.defaultFieldMap = defaultFieldMap == null ? null : Collections.unmodifiableMap(defaultFieldMap);
     }
 
     public TrainedModelConfig(StreamInput in) throws IOException {
@@ -157,6 +163,13 @@ public TrainedModelConfig(StreamInput in) throws IOException {
         estimatedHeapMemory = in.readVLong();
         estimatedOperations = in.readVLong();
         licenseLevel = License.OperationMode.parse(in.readString());
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+            this.defaultFieldMap = in.readBoolean() ?
+                Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString)) :
+                null;
+        } else {
+            this.defaultFieldMap = null;
+        }
     }
 
     public String getModelId() {
@@ -187,6 +200,10 @@ public Map<String, Object> getMetadata() {
         return metadata;
     }
 
+    public Map<String, String> getDefaultFieldMap() {
+        return defaultFieldMap;
+    }
+
     @Nullable
     public String getCompressedDefinition() throws IOException {
         if (definition == null) {
@@ -249,6 +266,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(estimatedHeapMemory);
         out.writeVLong(estimatedOperations);
         out.writeString(licenseLevel.description());
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+            if (defaultFieldMap != null) {
+                out.writeBoolean(true);
+                out.writeMap(defaultFieldMap, StreamOutput::writeString, StreamOutput::writeString);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
     }
 
     @Override
@@ -283,6 +308,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             new ByteSizeValue(estimatedHeapMemory));
         builder.field(ESTIMATED_OPERATIONS.getPreferredName(), estimatedOperations);
         builder.field(LICENSE_LEVEL.getPreferredName(), licenseLevel.description());
+        if (defaultFieldMap != null && defaultFieldMap.isEmpty() == false) {
+            builder.field(DEFAULT_FIELD_MAP.getPreferredName(), defaultFieldMap);
+        }
         builder.endObject();
         return builder;
     }
@@ -308,6 +336,7 @@ public boolean equals(Object o) {
             Objects.equals(estimatedHeapMemory, that.estimatedHeapMemory) &&
             Objects.equals(estimatedOperations, that.estimatedOperations) &&
             Objects.equals(licenseLevel, that.licenseLevel) &&
+            Objects.equals(defaultFieldMap, that.defaultFieldMap) &&
             Objects.equals(metadata, that.metadata);
     }
 
@@ -324,7 +353,8 @@ public int hashCode() {
             estimatedHeapMemory,
             estimatedOperations,
             input,
-            licenseLevel);
+            licenseLevel,
+            defaultFieldMap);
     }
 
     public static class Builder {
@@ -341,6 +371,7 @@ public static class Builder {
         private Long estimatedOperations;
         private LazyModelDefinition definition;
         private String licenseLevel;
+        private Map<String, String> defaultFieldMap;
 
         public Builder() {}
 
@@ -357,6 +388,7 @@ public Builder(TrainedModelConfig config) {
             this.estimatedOperations = config.estimatedOperations;
             this.estimatedHeapMemory = config.estimatedHeapMemory;
             this.licenseLevel = config.licenseLevel.description();
+            this.defaultFieldMap = config.defaultFieldMap == null ? null : new HashMap<>(config.defaultFieldMap);
         }
 
         public Builder setModelId(String modelId) {
@@ -475,6 +507,11 @@ public Builder setLicenseLevel(String licenseLevel) {
             return this;
         }
 
+        public Builder setDefaultFieldMap(Map<String, String> defaultFieldMap) {
+            this.defaultFieldMap = defaultFieldMap;
+            return this;
+        }
+
         public Builder validate() {
             return validate(false);
         }
@@ -567,7 +604,8 @@ public TrainedModelConfig build() {
                 input,
                 estimatedHeapMemory == null ? 0 : estimatedHeapMemory,
                 estimatedOperations == null ? 0 : estimatedOperations,
-                licenseLevel == null ? License.OperationMode.PLATINUM.description() : licenseLevel);
+                licenseLevel == null ? License.OperationMode.PLATINUM.description() : licenseLevel,
+                defaultFieldMap);
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
index f15d8296bd531..4a11c361d9457 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
@@ -53,11 +53,10 @@ private static ObjectParser<TrainedModelDefinition.Builder, Void> createParser(b
         ObjectParser<TrainedModelDefinition.Builder, Void> parser = new ObjectParser<>(NAME,
             ignoreUnknownFields,
             TrainedModelDefinition.Builder::builderForParser);
-        parser.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
+        parser.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
             (p, c, n) -> ignoreUnknownFields ?
                 p.namedObject(LenientlyParsedTrainedModel.class, n, null) :
                 p.namedObject(StrictlyParsedTrainedModel.class, n, null),
-            (modelDocBuilder) -> { /* Noop does not matter as we will throw if more than one is defined */ },
             TRAINED_MODEL);
         parser.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
             (p, c, n) -> ignoreUnknownFields ?
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
index 5c9996ba21588..5955ecda6f276 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -73,11 +73,10 @@ private static ObjectParser<Ensemble.Builder, Void> createParser(boolean lenient
                     p.namedObject(StrictlyParsedTrainedModel.class, n, null),
             (ensembleBuilder) -> ensembleBuilder.setModelsAreOrdered(true),
             TRAINED_MODELS);
-        parser.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
+        parser.declareNamedObject(Ensemble.Builder::setOutputAggregator,
             (p, c, n) ->
                 lenient ? p.namedObject(LenientlyParsedOutputAggregator.class, n, null) :
                     p.namedObject(StrictlyParsedOutputAggregator.class, n, null),
-            (ensembleBuilder) -> {/*Noop as it could be an array or object, it just has to be a one*/},
             AGGREGATE_OUTPUT);
         parser.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
         parser.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -414,14 +413,6 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
             return this;
         }
 
-        private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
-            if (outputAggregators.size() != 1) {
-                throw ExceptionsHelper.badRequestException("[{}] must have exactly one aggregator defined.",
-                    AGGREGATE_OUTPUT.getPreferredName());
-            }
-            this.setOutputAggregator(outputAggregators.get(0));
-        }
-
         private void setTargetType(String targetType) {
             this.targetType = TargetType.fromString(targetType);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
index 306ea243120e5..faa921971a267 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
@@ -13,10 +13,6 @@
 import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 
-import java.util.Comparator;
-import java.util.function.Predicate;
-import java.util.regex.Pattern;
-
 /**
  * Methods for handling index naming related functions
  */
@@ -27,31 +23,7 @@ public final class AnomalyDetectorsIndex {
     private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
     private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/";
 
-    // Visible for testing
-    static final Comparator<String> STATE_INDEX_NAME_COMPARATOR = new Comparator<>() {
-
-        private final Predicate<String> HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate();
-
-        @Override
-        public int compare(String index1, String index2) {
-            String[] index1Parts = index1.split("-");
-            String index1Suffix = index1Parts[index1Parts.length - 1];
-            boolean index1HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index1Suffix);
-            String[] index2Parts = index2.split("-");
-            String index2Suffix = index2Parts[index2Parts.length - 1];
-            boolean index2HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index2Suffix);
-            if (index1HasSixDigitsSuffix && index2HasSixDigitsSuffix) {
-                return index1Suffix.compareTo(index2Suffix);
-            } else if (index1HasSixDigitsSuffix != index2HasSixDigitsSuffix) {
-                return Boolean.compare(index1HasSixDigitsSuffix, index2HasSixDigitsSuffix);
-            } else {
-                return index1.compareTo(index2);
-            }
-        }
-    };
-
-    private AnomalyDetectorsIndex() {
-    }
+    private AnomalyDetectorsIndex() {}
 
     public static String jobResultsIndexPrefix() {
         return AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX;
@@ -108,8 +80,13 @@ public static String configIndexName() {
      */
     public static void createStateIndexAndAliasIfNecessary(Client client, ClusterState state, IndexNameExpressionResolver resolver,
                                                            final ActionListener<Boolean> finalListener) {
-        MlIndexAndAlias.createIndexAndAliasIfNecessary(client, state, resolver,
-            AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias(), finalListener);
+        MlIndexAndAlias.createIndexAndAliasIfNecessary(
+            client,
+            state,
+            resolver,
+            AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
+            AnomalyDetectorsIndex.jobStateIndexWriteAlias(),
+            finalListener);
     }
 
     public static String resultsMapping() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
index 9ae1aa2957adf..6a06608638ff9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
@@ -317,12 +317,13 @@ public final class ReservedFieldNames {
             Classification.NAME.getPreferredName(),
             Classification.DEPENDENT_VARIABLE.getPreferredName(),
             Classification.PREDICTION_FIELD_NAME.getPreferredName(),
+            Classification.CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(),
             Classification.NUM_TOP_CLASSES.getPreferredName(),
             Classification.TRAINING_PERCENT.getPreferredName(),
             BoostedTreeParams.LAMBDA.getPreferredName(),
             BoostedTreeParams.GAMMA.getPreferredName(),
             BoostedTreeParams.ETA.getPreferredName(),
-            BoostedTreeParams.MAXIMUM_NUMBER_TREES.getPreferredName(),
+            BoostedTreeParams.MAX_TREES.getPreferredName(),
             BoostedTreeParams.FEATURE_BAG_FRACTION.getPreferredName(),
             BoostedTreeParams.NUM_TOP_FEATURE_IMPORTANCE_VALUES.getPreferredName(),
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
index 79b9987d35193..2c3aba673c6d0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
@@ -5,20 +5,27 @@
  */
 package org.elasticsearch.xpack.core.ml.utils;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.Nullable;
 
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
@@ -30,6 +37,8 @@
  */
 public final class MlIndexAndAlias {
 
+    private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class);
+
     // Visible for testing
     static final Comparator<String> INDEX_NAME_COMPARATOR = new Comparator<>() {
 
@@ -61,60 +70,114 @@ private MlIndexAndAlias() {}
      * or to the index with the highest suffix if the index did not have to be created.
      * The listener is notified with a {@code boolean} that informs whether the index or the alias were created.
      */
-    public static void createIndexAndAliasIfNecessary(Client client, ClusterState clusterState, IndexNameExpressionResolver resolver,
-                                                      String indexPatternPrefix, String alias, ActionListener<Boolean> listener) {
-        if (clusterState.getMetaData().getAliasAndIndexLookup().containsKey(alias)) {
-            listener.onResponse(false);
-            return;
-        }
+    public static void createIndexAndAliasIfNecessary(Client client,
+                                                      ClusterState clusterState,
+                                                      IndexNameExpressionResolver resolver,
+                                                      String indexPatternPrefix,
+                                                      String alias,
+                                                      ActionListener<Boolean> listener) {
 
-        final ActionListener<String> createAliasListener = ActionListener.wrap(
-            concreteIndexName -> {
-                final IndicesAliasesRequest request = client.admin()
-                    .indices()
-                    .prepareAliases()
-                    .addAlias(concreteIndexName, alias)
-                    .request();
-                executeAsyncWithOrigin(client.threadPool().getThreadContext(),
-                    ML_ORIGIN,
-                    request,
-                    ActionListener.<AcknowledgedResponse>wrap(
-                        resp -> listener.onResponse(resp.isAcknowledged()),
-                        listener::onFailure),
-                    client.admin().indices()::aliases);
-            },
-            listener::onFailure
-        );
-
-        String[] stateIndices = resolver.concreteIndexNames(clusterState,
-            IndicesOptions.lenientExpandOpen(), indexPatternPrefix + "*");
-        if (stateIndices.length > 0) {
-            String latestStateIndex = Arrays.stream(stateIndices).max(INDEX_NAME_COMPARATOR).get();
-            createAliasListener.onResponse(latestStateIndex);
+        String legacyIndexWithoutSuffix = indexPatternPrefix;
+        String indexPattern = indexPatternPrefix + "*";
+        // The initial index name must be suitable for rollover functionality.
+        String firstConcreteIndex = indexPatternPrefix + "-000001";
+        String[] concreteIndexNames =
+            resolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), indexPattern);
+        Optional<IndexMetaData> indexPointedByCurrentWriteAlias = clusterState.getMetaData().hasAlias(alias)
+            ? clusterState.getMetaData().getAliasAndIndexLookup().get(alias).getIndices().stream().findFirst()
+            : Optional.empty();
+
+        if (concreteIndexNames.length == 0) {
+            if (indexPointedByCurrentWriteAlias.isEmpty()) {
+                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
+                return;
+            }
+            logger.error(
+                "There are no indices matching '{}' pattern but '{}' alias points at [{}]. This should never happen.",
+                indexPattern, alias, indexPointedByCurrentWriteAlias.get());
+        } else if (concreteIndexNames.length == 1 && concreteIndexNames[0].equals(legacyIndexWithoutSuffix)) {
+            if (indexPointedByCurrentWriteAlias.isEmpty()) {
+                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
+                return;
+            }
+            if (indexPointedByCurrentWriteAlias.get().getIndex().getName().equals(legacyIndexWithoutSuffix)) {
+                createFirstConcreteIndex(
+                    client,
+                    firstConcreteIndex,
+                    alias,
+                    false,
+                    ActionListener.wrap(
+                        unused -> updateWriteAlias(client, alias, legacyIndexWithoutSuffix, firstConcreteIndex, listener),
+                        listener::onFailure)
+                );
+                return;
+            }
+            logger.error(
+                "There is exactly one index (i.e. '{}') matching '{}' pattern but '{}' alias points at [{}]. This should never happen.",
+                legacyIndexWithoutSuffix, indexPattern, alias, indexPointedByCurrentWriteAlias.get());
         } else {
-            // The initial index name must be suitable for rollover functionality.
-            String initialJobStateIndex = indexPatternPrefix + "-000001";
-            CreateIndexRequest createIndexRequest = client.admin()
-                .indices()
-                .prepareCreate(initialJobStateIndex)
-                .addAlias(new Alias(alias))
-                .request();
-            executeAsyncWithOrigin(client.threadPool().getThreadContext(),
-                ML_ORIGIN,
-                createIndexRequest,
-                ActionListener.<CreateIndexResponse>wrap(
-                    createIndexResponse -> listener.onResponse(true),
-                    createIndexFailure -> {
-                        // If it was created between our last check, and this request being handled, we should add the alias
-                        // Adding an alias that already exists is idempotent. So, no need to double check if the alias exists
-                        // as well.
-                        if (ExceptionsHelper.unwrapCause(createIndexFailure) instanceof ResourceAlreadyExistsException) {
-                            createAliasListener.onResponse(initialJobStateIndex);
-                        } else {
-                            listener.onFailure(createIndexFailure);
-                        }
-                    }),
-                client.admin().indices()::create);
+            if (indexPointedByCurrentWriteAlias.isEmpty()) {
+                assert concreteIndexNames.length > 0;
+                String latestConcreteIndexName = Arrays.stream(concreteIndexNames).max(INDEX_NAME_COMPARATOR).get();
+                updateWriteAlias(client, alias, null, latestConcreteIndexName, listener);
+                return;
+            }
         }
+        // If the alias is set, there is nothing more to do.
+        listener.onResponse(false);
+    }
+
+    private static void createFirstConcreteIndex(Client client,
+                                                 String index,
+                                                 String alias,
+                                                 boolean addAlias,
+                                                 ActionListener<Boolean> listener) {
+        CreateIndexRequestBuilder requestBuilder = client.admin()
+            .indices()
+            .prepareCreate(index);
+        if (addAlias) {
+            requestBuilder.addAlias(new Alias(alias));
+        }
+        CreateIndexRequest request = requestBuilder.request();
+
+        executeAsyncWithOrigin(client.threadPool().getThreadContext(),
+            ML_ORIGIN,
+            request,
+            ActionListener.<CreateIndexResponse>wrap(
+                createIndexResponse -> listener.onResponse(true),
+                createIndexFailure -> {
+                    // If it was created between our last check, and this request being handled, we should add the alias
+                    // Adding an alias that already exists is idempotent. So, no need to double check if the alias exists
+                    // as well.
+                    if (ExceptionsHelper.unwrapCause(createIndexFailure) instanceof ResourceAlreadyExistsException) {
+                        updateWriteAlias(client, alias, null, index, listener);
+                    } else {
+                        listener.onFailure(createIndexFailure);
+                    }
+                }),
+            client.admin().indices()::create);
+    }
+
+    private static void updateWriteAlias(Client client,
+                                         String alias,
+                                         @Nullable String currentIndex,
+                                         String newIndex,
+                                         ActionListener<Boolean> listener) {
+        IndicesAliasesRequestBuilder requestBuilder = client.admin()
+            .indices()
+            .prepareAliases()
+            .addAlias(newIndex, alias);
+        if (currentIndex != null) {
+            requestBuilder.removeAlias(currentIndex, alias);
+        }
+        IndicesAliasesRequest request = requestBuilder.request();
+
+        executeAsyncWithOrigin(client.threadPool().getThreadContext(),
+            ML_ORIGIN,
+            request,
+            ActionListener.<AcknowledgedResponse>wrap(
+                resp -> listener.onResponse(resp.isAcknowledged()),
+                listener::onFailure),
+            client.admin().indices()::aliases);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
new file mode 100644
index 0000000000000..9828635ae64fe
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.search.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestStatus.OK;
+
+/**
+ * A response of an async search request.
+ */
+public class AsyncSearchResponse extends ActionResponse implements StatusToXContentObject {
+    @Nullable
+    private final String id;
+    private final int version;
+    private final SearchResponse searchResponse;
+    private final ElasticsearchException error;
+    private final boolean isRunning;
+    private final boolean isPartial;
+
+    private final long startTimeMillis;
+    private final long expirationTimeMillis;
+
+    /**
+     * Creates an {@link AsyncSearchResponse} with meta-information only (not-modified).
+     */
+    public AsyncSearchResponse(String id,
+                               int version,
+                               boolean isPartial,
+                               boolean isRunning,
+                               long startTimeMillis,
+                               long expirationTimeMillis) {
+        this(id, version, null, null, isPartial, isRunning, startTimeMillis, expirationTimeMillis);
+    }
+
+    /**
+     * Creates a new {@link AsyncSearchResponse}
+     *
+     * @param id The id of the search for further retrieval, <code>null</code> if not stored.
+     * @param version The version number of this response.
+     * @param searchResponse The actual search response.
+     * @param error The error if the search failed, <code>null</code> if the search is running
+     *                or has completed without failure.
+     * @param isPartial Whether the <code>searchResponse</code> contains partial results.
+     * @param isRunning Whether the search is running in the cluster.
+     * @param startTimeMillis The start date of the search in milliseconds since epoch.
+     */
+    public AsyncSearchResponse(String id,
+                               int version,
+                               SearchResponse searchResponse,
+                               ElasticsearchException error,
+                               boolean isPartial,
+                               boolean isRunning,
+                               long startTimeMillis,
+                               long expirationTimeMillis) {
+        this.id = id;
+        this.version = version;
+        this.error = error;
+        this.searchResponse = searchResponse;
+        this.isPartial = isPartial;
+        this.isRunning = isRunning;
+        this.startTimeMillis = startTimeMillis;
+        this.expirationTimeMillis = expirationTimeMillis;
+    }
+
+    public AsyncSearchResponse(StreamInput in) throws IOException {
+        this.id = in.readOptionalString();
+        this.version = in.readVInt();
+        this.error = in.readOptionalWriteable(ElasticsearchException::new);
+        this.searchResponse = in.readOptionalWriteable(SearchResponse::new);
+        this.isPartial = in.readBoolean();
+        this.isRunning = in.readBoolean();
+        this.startTimeMillis = in.readLong();
+        this.expirationTimeMillis = in.readLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(id);
+        out.writeVInt(version);
+        out.writeOptionalWriteable(error);
+        out.writeOptionalWriteable(searchResponse);
+        out.writeBoolean(isPartial);
+        out.writeBoolean(isRunning);
+        out.writeLong(startTimeMillis);
+        out.writeLong(expirationTimeMillis);
+    }
+
+    public AsyncSearchResponse clone(String id) {
+        return new AsyncSearchResponse(id, version, searchResponse, error, isPartial, false, startTimeMillis, expirationTimeMillis);
+    }
+
+    /**
+     * Returns the id of the async search request or null if the response is not stored in the cluster.
+     */
+    @Nullable
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Returns the version of this response.
+     */
+    public int getVersion() {
+        return version;
+    }
+
+    /**
+     * Returns the current {@link SearchResponse} or <code>null</code> if not available.
+     *
+     * See {@link #isPartial()} to determine whether the response contains partial or complete
+     * results.
+     */
+    public SearchResponse getSearchResponse() {
+        return searchResponse;
+    }
+
+    /**
+     * Returns the failure reason or null if the query is running or has completed normally.
+     */
+    public ElasticsearchException getFailure() {
+        return error;
+    }
+
+    /**
+     * Returns <code>true</code> if the {@link SearchResponse} contains partial
+     * results computed from a subset of the total shards.
+     */
+    public boolean isPartial() {
+        return isPartial;
+    }
+
+    /**
+     * Whether the search is still running in the cluster.
+     *
+     * A value of <code>false</code> indicates that the response is final
+     * even if {@link #isPartial()} returns <code>true</code>. In such case,
+     * the partial response represents the status of the search before a
+     * non-recoverable failure.
+     */
+    public boolean isRunning() {
+        return isRunning;
+    }
+
+    /**
+     * When this response was created as a timestamp in milliseconds since epoch.
+     */
+    public long getStartTime() {
+        return startTimeMillis;
+    }
+
+    /**
+     * When this response will expired as a timestamp in milliseconds since epoch.
+     */
+    public long getExpirationTime() {
+        return expirationTimeMillis;
+    }
+
+    @Override
+    public RestStatus status() {
+        if (searchResponse == null || isPartial) {
+            // shard failures are not considered fatal for partial results so
+            // we return OK until we get the final response even if we don't have
+            // a single successful shard.
+            return error != null ? error.status() : OK;
+        } else {
+            return searchResponse.status();
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (id != null) {
+            builder.field("id", id);
+        }
+        builder.field("version", version);
+        builder.field("is_partial", isPartial);
+        builder.field("is_running", isRunning);
+        builder.field("start_time_in_millis", startTimeMillis);
+        builder.field("expiration_time_in_millis", expirationTimeMillis);
+
+        if (searchResponse != null) {
+            builder.field("response");
+            searchResponse.toXContent(builder, params);
+        }
+        if (error != null) {
+            builder.startObject("error");
+            error.toXContent(builder, params);
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java
new file mode 100644
index 0000000000000..d69de80d2293e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.search.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DeleteAsyncSearchAction extends ActionType<AcknowledgedResponse> {
+    public static final DeleteAsyncSearchAction INSTANCE = new DeleteAsyncSearchAction();
+    public static final String NAME = "indices:data/read/async_search/delete";
+
+    private DeleteAsyncSearchAction() {
+        super(NAME, AcknowledgedResponse::new);
+    }
+
+    @Override
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
+    }
+
+    public static class Request extends ActionRequest {
+        private final String id;
+
+        public Request(String id) {
+            this.id = id;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.id = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(id);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return id.equals(request.id);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java
new file mode 100644
index 0000000000000..fe4801aab4a10
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.search.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest.MIN_KEEP_ALIVE;
+
+public class GetAsyncSearchAction extends ActionType<AsyncSearchResponse> {
+    public static final GetAsyncSearchAction INSTANCE = new GetAsyncSearchAction();
+    public static final String NAME = "indices:data/read/async_search/get";
+
+    private GetAsyncSearchAction() {
+        super(NAME, AsyncSearchResponse::new);
+    }
+
+    @Override
+    public Writeable.Reader<AsyncSearchResponse> getResponseReader() {
+        return AsyncSearchResponse::new;
+    }
+
+    public static class Request extends ActionRequest {
+        private final String id;
+        private TimeValue waitForCompletion = TimeValue.MINUS_ONE;
+        private TimeValue keepAlive = TimeValue.MINUS_ONE;
+
+        /**
+         * Creates a new request
+         *
+         * @param id The id of the search progress request.
+         */
+        public Request(String id) {
+            this.id = id;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.id = in.readString();
+            this.waitForCompletion = TimeValue.timeValueMillis(in.readLong());
+            this.keepAlive = in.readTimeValue();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(id);
+            out.writeLong(waitForCompletion.millis());
+            out.writeTimeValue(keepAlive);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+            if (keepAlive.getMillis() != -1 && keepAlive.getMillis() < MIN_KEEP_ALIVE) {
+                validationException =
+                    addValidationError("keep_alive must be greater than 1 minute, got:" + keepAlive.toString(), validationException);
+            }
+            return validationException;
+        }
+
+        /**
+         * Returns the id of the async search.
+         */
+        public String getId() {
+            return id;
+        }
+
+        /**
+         * Sets the minimum time that the request should wait before returning a partial result (defaults to no wait).
+         */
+        public Request setWaitForCompletion(TimeValue timeValue) {
+            this.waitForCompletion = timeValue;
+            return this;
+        }
+
+        public TimeValue getWaitForCompletion() {
+            return waitForCompletion;
+        }
+
+        /**
+         * Extends the amount of time after which the result will expire (defaults to no extension).
+         */
+        public Request setKeepAlive(TimeValue timeValue) {
+            this.keepAlive = timeValue;
+            return this;
+        }
+
+        public TimeValue getKeepAlive() {
+            return keepAlive;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Objects.equals(id, request.id) &&
+                waitForCompletion.equals(request.waitForCompletion) &&
+                keepAlive.equals(request.keepAlive);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, waitForCompletion, keepAlive);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java
new file mode 100644
index 0000000000000..a91c861c64e16
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.search.action;
+
+import org.elasticsearch.action.ActionType;
+
+public final class SubmitAsyncSearchAction extends ActionType<AsyncSearchResponse> {
+    public static final SubmitAsyncSearchAction INSTANCE = new SubmitAsyncSearchAction();
+    public static final String NAME = "indices:data/read/async_search/submit";
+
+    private SubmitAsyncSearchAction() {
+        super(NAME, AsyncSearchResponse::new);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
new file mode 100644
index 0000000000000..a6397cbf08c3b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.search.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request to track asynchronously the progress of a search against one or more indices.
+ *
+ * @see AsyncSearchResponse
+ */
+public class SubmitAsyncSearchRequest extends ActionRequest {
+    public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
+
+    private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1);
+    private boolean cleanOnCompletion = true;
+    private TimeValue keepAlive = TimeValue.timeValueDays(5);
+
+    private final SearchRequest request;
+
+    /**
+     * Creates a new request
+     */
+    public SubmitAsyncSearchRequest(String... indices) {
+        this(new SearchSourceBuilder(), indices);
+    }
+
+    /**
+     * Creates a new request
+     */
+    public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) {
+        this.request = new SearchRequest(indices, source);
+        request.setCcsMinimizeRoundtrips(false);
+        request.setPreFilterShardSize(1);
+        request.setBatchedReduceSize(5);
+        request.requestCache(true);
+    }
+
+    public SubmitAsyncSearchRequest(StreamInput in) throws IOException {
+        this.request = new SearchRequest(in);
+        this.waitForCompletion = in.readTimeValue();
+        this.keepAlive = in.readTimeValue();
+        this.cleanOnCompletion = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        request.writeTo(out);
+        out.writeTimeValue(waitForCompletion);
+        out.writeTimeValue(keepAlive);
+        out.writeBoolean(cleanOnCompletion);
+    }
+
+    /**
+     * Sets the number of shard results that should be returned to notify search progress (default to 5).
+     */
+    public SubmitAsyncSearchRequest setBatchedReduceSize(int size) {
+        request.setBatchedReduceSize(size);
+        return this;
+    }
+
+    public int getBatchReduceSize() {
+        return request.getBatchedReduceSize();
+    }
+
+    /**
+     * Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second).
+     */
+    public SubmitAsyncSearchRequest setWaitForCompletion(TimeValue waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+        return this;
+    }
+
+    public TimeValue getWaitForCompletion() {
+        return waitForCompletion;
+    }
+
+    /**
+     * Sets the amount of time after which the result will expire (defaults to 5 days).
+     */
+    public SubmitAsyncSearchRequest setKeepAlive(TimeValue keepAlive) {
+        this.keepAlive = keepAlive;
+        return this;
+    }
+
+    public TimeValue getKeepAlive() {
+        return keepAlive;
+    }
+
+    /**
+     * Returns the underlying {@link SearchRequest}.
+     */
+    public SearchRequest getSearchRequest() {
+        return request;
+    }
+
+    /**
+     * Should the resource be removed on completion or failure (defaults to true).
+     */
+    public SubmitAsyncSearchRequest setCleanOnCompletion(boolean value) {
+        this.cleanOnCompletion = value;
+        return this;
+    }
+
+    public boolean isCleanOnCompletion() {
+        return cleanOnCompletion;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = request.validate();
+        if (request.scroll() != null) {
+            addValidationError("[scroll] queries are not supported", validationException);
+        }
+        if (request.isSuggestOnly()) {
+            validationException = addValidationError("suggest-only queries are not supported", validationException);
+        }
+        if (keepAlive.getMillis() < MIN_KEEP_ALIVE) {
+            validationException =
+                addValidationError("[keep_alive] must be greater than 1 minute, got:" + keepAlive.toString(), validationException);
+        }
+        if (request.isCcsMinimizeRoundtrips()) {
+            validationException =
+                addValidationError("[ccs_minimize_roundtrips] is not supported on async search queries", validationException);
+        }
+
+        return validationException;
+    }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, toString(), parentTaskId, headers) {
+            @Override
+            public boolean shouldCancelChildrenOnCancellation() {
+                return true;
+            }
+        };
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SubmitAsyncSearchRequest request1 = (SubmitAsyncSearchRequest) o;
+        return cleanOnCompletion == request1.cleanOnCompletion &&
+            waitForCompletion.equals(request1.waitForCompletion) &&
+            keepAlive.equals(request1.keepAlive) &&
+            request.equals(request1.request);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(waitForCompletion, cleanOnCompletion, keepAlive, request);
+    }
+
+    @Override
+    public String toString() {
+        return "SubmitAsyncSearchRequest{" +
+            "waitForCompletion=" + waitForCompletion +
+            ", cleanOnCompletion=" + cleanOnCompletion +
+            ", keepAlive=" + keepAlive +
+            ", request=" + request +
+            '}';
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
index c31dbeb7e4859..67307ad88cf3b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
@@ -74,11 +74,15 @@ public CreateApiKeyRequestBuilder source(BytesReference source, XContentType xCo
         final NamedXContentRegistry registry = NamedXContentRegistry.EMPTY;
         try (InputStream stream = source.streamInput();
                 XContentParser parser = xContentType.xContent().createParser(registry, LoggingDeprecationHandler.INSTANCE, stream)) {
-            CreateApiKeyRequest createApiKeyRequest = PARSER.parse(parser, null);
+            CreateApiKeyRequest createApiKeyRequest = parse(parser);
             setName(createApiKeyRequest.getName());
             setRoleDescriptors(createApiKeyRequest.getRoleDescriptors());
             setExpiration(createApiKeyRequest.getExpiration());
         }
         return this;
     }
+
+    public static CreateApiKeyRequest parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java
new file mode 100644
index 0000000000000..07e5c3bbf2521
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyAction.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionType;
+
+/**
+ * ActionType for the creation of an API key on behalf of another user
+ * This returns the {@link CreateApiKeyResponse} because the REST output is intended to be identical to the {@link CreateApiKeyAction}.
+ */
+public final class GrantApiKeyAction extends ActionType<CreateApiKeyResponse> {
+
+    public static final String NAME = "cluster:admin/xpack/security/api_key/grant";
+    public static final GrantApiKeyAction INSTANCE = new GrantApiKeyAction();
+
+    private GrantApiKeyAction() {
+        super(NAME, CreateApiKeyResponse::new);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java
new file mode 100644
index 0000000000000..4daf3c84df61c
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GrantApiKeyRequest.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.SecureString;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request class used for the creation of an API key on behalf of another user.
+ * Logically this is similar to {@link CreateApiKeyRequest}, but is for cases when the user that has permission to call this action
+ * is different to the user for whom the API key should be created
+ */
+public final class GrantApiKeyRequest extends ActionRequest {
+
+    public static final String PASSWORD_GRANT_TYPE = "password";
+    public static final String ACCESS_TOKEN_GRANT_TYPE = "access_token";
+
+    /**
+     * Fields related to the end user authentication
+     */
+    public static class Grant implements Writeable {
+        private String type;
+        private String username;
+        private SecureString password;
+        private SecureString accessToken;
+
+        public Grant() {
+        }
+
+        public Grant(StreamInput in) throws IOException {
+            this.type = in.readString();
+            this.username = in.readOptionalString();
+            this.password = in.readOptionalSecureString();
+            this.accessToken = in.readOptionalSecureString();
+        }
+
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(type);
+            out.writeOptionalString(username);
+            out.writeOptionalSecureString(password);
+            out.writeOptionalSecureString(accessToken);
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        public String getUsername() {
+            return username;
+        }
+
+        public SecureString getPassword() {
+            return password;
+        }
+
+        public SecureString getAccessToken() {
+            return accessToken;
+        }
+
+        public void setType(String type) {
+            this.type = type;
+        }
+
+        public void setUsername(String username) {
+            this.username = username;
+        }
+
+        public void setPassword(SecureString password) {
+            this.password = password;
+        }
+
+        public void setAccessToken(SecureString accessToken) {
+            this.accessToken = accessToken;
+        }
+    }
+
+    private final Grant grant;
+    private CreateApiKeyRequest apiKey;
+
+    public GrantApiKeyRequest() {
+        this.grant = new Grant();
+        this.apiKey = new CreateApiKeyRequest();
+    }
+
+    public GrantApiKeyRequest(StreamInput in) throws IOException {
+        super(in);
+        this.grant = new Grant(in);
+        this.apiKey = new CreateApiKeyRequest(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        grant.writeTo(out);
+        apiKey.writeTo(out);
+    }
+
+    public WriteRequest.RefreshPolicy getRefreshPolicy() {
+        return apiKey.getRefreshPolicy();
+    }
+
+    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
+        apiKey.setRefreshPolicy(refreshPolicy);
+    }
+
+    public Grant getGrant() {
+        return grant;
+    }
+
+    public CreateApiKeyRequest getApiKeyRequest() {
+        return apiKey;
+    }
+
+    public void setApiKeyRequest(CreateApiKeyRequest apiKeyRequest) {
+        this.apiKey = Objects.requireNonNull(apiKeyRequest, "Cannot set a null api_key");
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = apiKey.validate();
+        if (grant.type == null) {
+            validationException = addValidationError("[grant_type] is required", validationException);
+        } else if (grant.type.equals(PASSWORD_GRANT_TYPE)) {
+            validationException = validateRequiredField("username", grant.username, validationException);
+            validationException = validateRequiredField("password", grant.password, validationException);
+            validationException = validateUnsupportedField("access_token", grant.accessToken, validationException);
+        } else if (grant.type.equals(ACCESS_TOKEN_GRANT_TYPE)) {
+            validationException = validateRequiredField("access_token", grant.accessToken, validationException);
+            validationException = validateUnsupportedField("username", grant.username, validationException);
+            validationException = validateUnsupportedField("password", grant.password, validationException);
+        } else {
+            validationException = addValidationError("grant_type [" + grant.type + "] is not supported", validationException);
+        }
+        return validationException;
+    }
+
+    private ActionRequestValidationException validateRequiredField(String fieldName, CharSequence fieldValue,
+                                                                   ActionRequestValidationException validationException) {
+        if (fieldValue == null || fieldValue.length() == 0) {
+            return addValidationError("[" + fieldName + "] is required for grant_type [" + grant.type + "]", validationException);
+        }
+        return validationException;
+    }
+
+    private ActionRequestValidationException validateUnsupportedField(String fieldName, CharSequence fieldValue,
+                                                                      ActionRequestValidationException validationException) {
+        if (fieldValue != null && fieldValue.length() > 0) {
+            return addValidationError("[" + fieldName + "] is not supported for grant_type [" + grant.type + "]", validationException);
+        }
+        return validationException;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java
index 77f6c537b6f81..2d7f660bd188e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java
@@ -23,7 +23,7 @@ public final class RestrictedIndicesNames {
     public static final String SECURITY_TOKENS_ALIAS = ".security-tokens";
 
     // public for tests
-    public static final String ASYNC_SEARCH_PREFIX = ".async-search-";
+    public static final String ASYNC_SEARCH_PREFIX = ".async-search";
     private static final Automaton ASYNC_SEARCH_AUTOMATON = Automatons.patterns(ASYNC_SEARCH_PREFIX + "*");
 
     // public for tests
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
index 0ef7912c58017..0d7ab44ed9582 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
@@ -254,10 +254,10 @@ public String generateSnapshotName(Context context) {
      */
     public CreateSnapshotRequest toRequest() {
         CreateSnapshotRequest req = new CreateSnapshotRequest(repository, generateSnapshotName(new ResolverContext()));
+        Map<String, Object> mergedConfiguration = configuration == null ? new HashMap<>() : new HashMap<>(configuration);
         @SuppressWarnings("unchecked")
-        Map<String, Object> metadata = (Map<String, Object>) configuration.get("metadata");
+        Map<String, Object> metadata = (Map<String, Object>) mergedConfiguration.get("metadata");
         Map<String, Object> metadataWithAddedPolicyName = addPolicyNameToMetadata(metadata);
-        Map<String, Object> mergedConfiguration = new HashMap<>(configuration);
         mergedConfiguration.put("metadata", metadataWithAddedPolicyName);
         req.source(mergedConfiguration);
         req.waitForCompletion(true);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java
index 5686f3c9377e1..bd215126119d5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/SingleGroupSource.java
@@ -80,7 +80,7 @@ public SingleGroupSource(final String field, final ScriptConfig scriptConfig) {
 
     public SingleGroupSource(StreamInput in) throws IOException {
         field = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // change to 7.7.0
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
             scriptConfig = in.readOptionalWriteable(ScriptConfig::new);
         } else {
             scriptConfig = null;
@@ -107,7 +107,7 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(field);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
             out.writeOptionalWriteable(scriptConfig);
         }
     }
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json
index fb0b7ad6c2bd5..e3cd7f51dc43a 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json
@@ -9,6 +9,7 @@
       "auto_expand_replicas" : "0-1",
       "hidden": true
     }
+    ${xpack.ml.index.lifecycle.settings}
   },
   "mappings" : {
     "_doc": {
@@ -18,5 +19,5 @@
       "enabled": false
     }
   },
-  "aliases" : { }
+  "aliases" : {}
 }
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
index 1abd7e42e8ad1..fe0c559b12245 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
@@ -40,9 +40,12 @@
               "lambda" : {
                 "type" : "double"
               },
-              "maximum_number_trees" : {
+              "max_trees" : {
                 "type" : "integer"
               },
+              "class_assignment_objective" : {
+                "type" : "keyword"
+              },
               "num_top_classes" : {
                 "type" : "integer"
               },
@@ -87,7 +90,7 @@
               "lambda" : {
                 "type" : "double"
               },
-              "maximum_number_trees" : {
+              "max_trees" : {
                 "type" : "integer"
               },
               "num_top_feature_importance_values" : {
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json
index 5cbee23ca8094..2a1131ca557a9 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json
@@ -64,6 +64,9 @@
         },
         "total_definition_length": {
           "type": "long"
+        },
+        "default_field_map": {
+          "enabled": false
         }
       }
     }
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json
new file mode 100644
index 0000000000000..09a8db4e1ee9d
--- /dev/null
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json
@@ -0,0 +1,11 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_size": "50GB"
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json
index 1c694d9d1a7e5..223d1a79bad75 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json
@@ -10,6 +10,8 @@
       "auto_expand_replicas" : "0-1",
       "hidden": true
     }
+    ${xpack.ml.index.lifecycle.settings}
   },
-  "mappings" : ${xpack.ml.stats.mappings}
+  "mappings" : ${xpack.ml.stats.mappings},
+  "aliases" : {}
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
index ef4d35401675b..c3958f31563d0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
@@ -105,9 +105,9 @@ public void testSnapshotAndRestore() throws Exception {
         });
         assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
 
-        e = expectThrows(SearchPhaseExecutionException.class, () ->
-            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get());
-        assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
+        // can-match phase pre-filters access to non-existing field
+        assertEquals(0,
+            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get().getHits().getTotalHits().value);
         // make sure deletes do not work
         String idToDelete = "" + randomIntBetween(0, builders.length);
         expectThrows(ClusterBlockException.class, () -> client().prepareDelete(sourceIdx, idToDelete)
@@ -130,9 +130,9 @@ public void testSnapshotAndRestoreWithNested() throws Exception {
         SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () ->
             client().prepareSearch(sourceIdx).setQuery(QueryBuilders.idsQuery().addIds("" + randomIntBetween(0, builders.length))).get());
         assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
-        e = expectThrows(SearchPhaseExecutionException.class, () ->
-            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get());
-        assertTrue(e.toString().contains("_source only indices can't be searched or filtered"));
+        // can-match phase pre-filters access to non-existing field
+        assertEquals(0,
+            client().prepareSearch(sourceIdx).setQuery(QueryBuilders.termQuery("field1", "bar")).get().getHits().getTotalHits().value);
         // make sure deletes do not work
         String idToDelete = "" + randomIntBetween(0, builders.length);
         expectThrows(ClusterBlockException.class, () -> client().prepareDelete(sourceIdx, idToDelete)
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
index 8cd8a2ef0b36d..8dad3cf3a4a81 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.protocol.xpack.frozen.FreezeRequest;
+import org.elasticsearch.protocol.xpack.frozen.FreezeResponse;
 import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
 import org.elasticsearch.xpack.core.ilm.Step.StepKey;
 import org.mockito.Mockito;
@@ -73,7 +74,7 @@ public void testFreeze() {
             assertNotNull(request);
             assertEquals(1, request.indices().length);
             assertEquals(indexMetaData.getIndex().getName(), request.indices()[0]);
-            listener.onResponse(null);
+            listener.onResponse(new FreezeResponse(true, true));
             return null;
         }).when(indicesClient).execute(Mockito.any(), Mockito.any(), Mockito.any());
 
@@ -127,4 +128,32 @@ public void onFailure(Exception e) {
 
         assertThat(exceptionThrown.get(), equalTo(true));
     }
+
+    public void testNotAcknowledged() {
+        IndexMetaData indexMetaData = getIndexMetaData();
+
+        Mockito.doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[2];
+            listener.onResponse(new FreezeResponse(false, false));
+            return null;
+        }).when(indicesClient).execute(Mockito.any(), Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        FreezeStep step = createRandomInstance();
+        step.performAction(indexMetaData, emptyClusterState(), null, new AsyncActionStep.Listener() {
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertEquals("freeze index request failed to be acknowledged", e.getMessage());
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertThat(exceptionThrown.get(), equalTo(true));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
index 7dd0f6f9b0ca9..1c733e04504c8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
@@ -155,12 +155,14 @@ protected void assertOnBWCObject(DataFrameAnalyticsConfig bwcSerializedObject, D
             bwcAnalysis = new Classification(bwcClassification.getDependentVariable(),
                 bwcClassification.getBoostedTreeParams(),
                 bwcClassification.getPredictionFieldName(),
+                bwcClassification.getClassAssignmentObjective(),
                 bwcClassification.getNumTopClasses(),
                 bwcClassification.getTrainingPercent(),
                 42L);
             testAnalysis = new Classification(testClassification.getDependentVariable(),
                 testClassification.getBoostedTreeParams(),
                 testClassification.getPredictionFieldName(),
+                testClassification.getClassAssignmentObjective(),
                 testClassification.getNumTopClasses(),
                 testClassification.getTrainingPercent(),
                 42L);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
index 7f9d8c243e50b..f1114bf3445a8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
@@ -39,7 +39,7 @@ public static BoostedTreeParams createRandom() {
             .setLambda(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setGamma(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
             .setEta(randomBoolean() ? null : randomDoubleBetween(0.001, 1.0, true))
-            .setMaximumNumberTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
+            .setMaxTrees(randomBoolean() ? null : randomIntBetween(1, 2000))
             .setFeatureBagFraction(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, false))
             .setNumTopFeatureImportanceValues(randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE))
             .build();
@@ -88,16 +88,16 @@ public void testConstructor_GivenEtaIsGreaterThanOne() {
 
     public void testConstructor_GivenMaximumNumberTreesIsZero() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> BoostedTreeParams.builder().setMaximumNumberTrees(0).build());
+            () -> BoostedTreeParams.builder().setMaxTrees(0).build());
 
-        assertThat(e.getMessage(), equalTo("[maximum_number_trees] must be an integer in [1, 2000]"));
+        assertThat(e.getMessage(), equalTo("[max_trees] must be an integer in [1, 2000]"));
     }
 
     public void testConstructor_GivenMaximumNumberTreesIsGreaterThan2k() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> BoostedTreeParams.builder().setMaximumNumberTrees(2001).build());
+            () -> BoostedTreeParams.builder().setMaxTrees(2001).build());
 
-        assertThat(e.getMessage(), equalTo("[maximum_number_trees] must be an integer in [1, 2000]"));
+        assertThat(e.getMessage(), equalTo("[max_trees] must be an integer in [1, 2000]"));
     }
 
     public void testConstructor_GivenFeatureBagFractionIsLessThanZero() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index 0898a3fb476e8..7429fe08d180d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -54,17 +54,20 @@ public static Classification createRandom() {
         String dependentVariableName = randomAlphaOfLength(10);
         BoostedTreeParams boostedTreeParams = BoostedTreeParamsTests.createRandom();
         String predictionFieldName = randomBoolean() ? null : randomAlphaOfLength(10);
+        Classification.ClassAssignmentObjective classAssignmentObjective = randomBoolean() ?
+            null : randomFrom(Classification.ClassAssignmentObjective.values());
         Integer numTopClasses = randomBoolean() ? null : randomIntBetween(0, 1000);
         Double trainingPercent = randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true);
         Long randomizeSeed = randomBoolean() ? null : randomLong();
-        return new Classification(dependentVariableName, boostedTreeParams, predictionFieldName, numTopClasses, trainingPercent,
-            randomizeSeed);
+        return new Classification(dependentVariableName, boostedTreeParams, predictionFieldName, classAssignmentObjective,
+            numTopClasses, trainingPercent, randomizeSeed);
     }
 
     public static Classification mutateForVersion(Classification instance, Version version) {
         return new Classification(instance.getDependentVariable(),
             BoostedTreeParamsTests.mutateForVersion(instance.getBoostedTreeParams(), version),
             instance.getPredictionFieldName(),
+            version.onOrAfter(Version.V_8_0_0) ? instance.getClassAssignmentObjective() : null,
             instance.getNumTopClasses(),
             instance.getTrainingPercent(),
             instance.getRandomizeSeed());
@@ -80,12 +83,14 @@ protected void assertOnBWCObject(Classification bwcSerializedObject, Classificat
         Classification newBwc = new Classification(bwcSerializedObject.getDependentVariable(),
             bwcSerializedObject.getBoostedTreeParams(),
             bwcSerializedObject.getPredictionFieldName(),
+            bwcSerializedObject.getClassAssignmentObjective(),
             bwcSerializedObject.getNumTopClasses(),
             bwcSerializedObject.getTrainingPercent(),
             42L);
         Classification newInstance = new Classification(testInstance.getDependentVariable(),
             testInstance.getBoostedTreeParams(),
             testInstance.getPredictionFieldName(),
+            testInstance.getClassAssignmentObjective(),
             testInstance.getNumTopClasses(),
             testInstance.getTrainingPercent(),
             42L);
@@ -99,104 +104,129 @@ protected Writeable.Reader<Classification> instanceReader() {
 
     public void testConstructor_GivenTrainingPercentIsLessThanOne() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 0.999, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 0.999, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[training_percent] must be a double in [1, 100]"));
     }
 
     public void testConstructor_GivenTrainingPercentIsGreaterThan100() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 100.0001, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 100.0001, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[training_percent] must be a double in [1, 100]"));
     }
 
     public void testConstructor_GivenNumTopClassesIsLessThanZero() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", -1, 1.0, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, -1, 1.0, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[num_top_classes] must be an integer in [0, 1000]"));
     }
 
     public void testConstructor_GivenNumTopClassesIsGreaterThan1000() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 1001, 1.0, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1001, 1.0, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[num_top_classes] must be an integer in [0, 1000]"));
     }
 
     public void testGetPredictionFieldName() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 50.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 50.0, randomLong());
         assertThat(classification.getPredictionFieldName(), equalTo("result"));
 
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, null, 3, 50.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, null, null, 3, 50.0, randomLong());
         assertThat(classification.getPredictionFieldName(), equalTo("foo_prediction"));
     }
 
+    public void testClassAssignmentObjective() {
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result",
+            Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY, 7, 1.0, randomLong());
+        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY));
+
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result",
+        Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL, 7, 1.0, randomLong());
+        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL));
+
+        // class_assignment_objective == null, default applied
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 7, 1.0, randomLong());
+        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL));
+    }
+
     public void testGetNumTopClasses() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 7, 1.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 7, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(7));
 
         // Boundary condition: num_top_classes == 0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 0, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 0, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(0));
 
         // Boundary condition: num_top_classes == 1000
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 1000, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1000, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(1000));
 
         // num_top_classes == null, default applied
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, null, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(2));
     }
 
     public void testGetTrainingPercent() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 50.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 50.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(50.0));
 
         // Boundary condition: training_percent == 1.0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 1.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(1.0));
 
         // Boundary condition: training_percent == 100.0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 100.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 100.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(100.0));
 
         // training_percent == null, default applied
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, null, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, null, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(100.0));
     }
 
     public void testGetParams() {
-        Map<String, Set<String>> extractedFields =
+        DataFrameAnalysis.FieldInfo fieldInfo = new TestFieldInfo(
             Map.of(
                 "foo", Set.of(BooleanFieldMapper.CONTENT_TYPE),
                 "bar", Set.of(NumberFieldMapper.NumberType.LONG.typeName()),
-                "baz", Set.of(KeywordFieldMapper.CONTENT_TYPE));
+                "baz", Set.of(KeywordFieldMapper.CONTENT_TYPE)),
+            Map.of(
+                "foo", 10L,
+                "bar", 20L,
+                "baz", 30L)
+        );
         assertThat(
-            new Classification("foo").getParams(extractedFields),
+            new Classification("foo").getParams(fieldInfo),
             equalTo(
                 Map.of(
                     "dependent_variable", "foo",
+                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "foo_prediction",
-                    "prediction_field_type", "bool")));
+                    "prediction_field_type", "bool",
+                    "num_classes", 10L)));
         assertThat(
-            new Classification("bar").getParams(extractedFields),
+            new Classification("bar").getParams(fieldInfo),
             equalTo(
                 Map.of(
                     "dependent_variable", "bar",
+                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "bar_prediction",
-                    "prediction_field_type", "int")));
+                    "prediction_field_type", "int",
+                    "num_classes", 20L)));
         assertThat(
-            new Classification("baz").getParams(extractedFields),
+            new Classification("baz").getParams(fieldInfo),
             equalTo(
                 Map.of(
                     "dependent_variable", "baz",
+                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "baz_prediction",
-                    "prediction_field_type", "string")));
+                    "prediction_field_type", "string",
+                    "num_classes", 30L)));
     }
 
     public void testRequiredFieldsIsNonEmpty() {
@@ -210,7 +240,7 @@ public void testFieldCardinalityLimitsIsNonEmpty() {
         assertThat(constraints.size(), equalTo(1));
         assertThat(constraints.get(0).getField(), equalTo(classification.getDependentVariable()));
         assertThat(constraints.get(0).getLowerBound(), equalTo(2L));
-        assertThat(constraints.get(0).getUpperBound(), equalTo(2L));
+        assertThat(constraints.get(0).getUpperBound(), equalTo(30L));
     }
 
     public void testGetExplicitlyMappedFields() {
@@ -309,4 +339,25 @@ public void testExtractJobIdFromStateDoc() {
     protected Classification mutateInstanceForVersion(Classification instance, Version version) {
         return mutateForVersion(instance, version);
     }
+
+    private static class TestFieldInfo implements DataFrameAnalysis.FieldInfo {
+
+        private final Map<String, Set<String>> fieldTypes;
+        private final Map<String, Long> fieldCardinalities;
+
+        private TestFieldInfo(Map<String, Set<String>> fieldTypes, Map<String, Long> fieldCardinalities) {
+            this.fieldTypes = fieldTypes;
+            this.fieldCardinalities = fieldCardinalities;
+        }
+
+        @Override
+        public Set<String> getTypes(String field) {
+            return fieldTypes.get(field);
+        }
+
+        @Override
+        public Long getCardinality(String field) {
+            return fieldCardinalities.get(field);
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java
new file mode 100644
index 0000000000000..1efe45fa930e4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationParametersTests.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EvaluationParametersTests extends ESTestCase {
+
+    public void testConstructorAndGetters() {
+        EvaluationParameters params = new EvaluationParameters(17);
+        assertThat(params.getMaxBuckets(), equalTo(17));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
index 16609dbdddcb0..1f755a93aad4a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
@@ -10,25 +10,28 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.PerClassResult;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.Result;
 
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockCardinality;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFilters;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFiltersBucket;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockSingleValue;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTermsBucket;
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
 public class AccuracyTests extends AbstractSerializingTestCase<Accuracy> {
 
+    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
+
     @Override
     protected Accuracy doParseInstance(XContentParser parser) throws IOException {
         return Accuracy.fromXContent(parser);
@@ -61,6 +64,7 @@ public void testProcess() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 100L),
+            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 1000L),
             mockFilters(
                 "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -76,13 +80,12 @@ public void testProcess() {
                         new Aggregations(List.of(mockFilters(
                             "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
                             List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L)))))))),
-            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 1000L),
             mockSingleValue(Accuracy.OVERALL_ACCURACY_AGG_NAME, 0.5)));
 
         Accuracy accuracy = new Accuracy();
         accuracy.process(aggs);
 
-        assertThat(accuracy.aggs("act", "pred"), isTuple(empty(), empty()));
+        assertThat(accuracy.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
 
         Result result = accuracy.getResult().get();
         assertThat(result.getMetricName(), equalTo(Accuracy.NAME.getPreferredName()));
@@ -103,6 +106,7 @@ public void testProcess_GivenCardinalityTooHigh() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 100L),
+            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 1001L),
             mockFilters(
                 "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -118,11 +122,10 @@ public void testProcess_GivenCardinalityTooHigh() {
                         new Aggregations(List.of(mockFilters(
                             "accuracy_" + MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
                             List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L)))))))),
-            mockCardinality("accuracy_" + MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 1001L),
             mockSingleValue(Accuracy.OVERALL_ACCURACY_AGG_NAME, 0.5)));
 
         Accuracy accuracy = new Accuracy();
-        accuracy.aggs("foo", "bar");
+        accuracy.aggs(EVALUATION_PARAMETERS, "foo", "bar");
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> accuracy.process(aggs));
         assertThat(e.getMessage(), containsString("Cardinality of field [foo] is too high"));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
index f205974809961..bad673e28d7c9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 
 import java.io.IOException;
@@ -43,6 +44,8 @@
 
 public class ClassificationTests extends AbstractSerializingTestCase<Classification> {
 
+    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(MlEvaluationNamedXContentProvider.getNamedWriteables());
@@ -100,7 +103,7 @@ public void testBuildSearch() {
 
         Classification evaluation = new Classification("act", "pred", Arrays.asList(new MulticlassConfusionMatrix()));
 
-        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(userProvidedQuery);
+        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(EVALUATION_PARAMETERS, userProvidedQuery);
         assertThat(searchSourceBuilder.query(), equalTo(expectedSearchQuery));
         assertThat(searchSourceBuilder.aggregations().count(), greaterThan(0));
     }
@@ -196,7 +199,9 @@ public String getWriteableName() {
         }
 
         @Override
-        public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(String actualField, String predictedField) {
+        public Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs(EvaluationParameters parameters,
+                                                                                      String actualField,
+                                                                                      String predictedField) {
             return Tuple.tuple(List.of(), List.of());
         }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
index 98697d3536f32..642bb9d915b17 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix.ActualClass;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix.PredictedClass;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix.Result;
@@ -21,18 +22,20 @@
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty;
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockCardinality;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFilters;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFiltersBucket;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTermsBucket;
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
 public class MulticlassConfusionMatrixTests extends AbstractSerializingTestCase<MulticlassConfusionMatrix> {
 
+    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
+
     @Override
     protected MulticlassConfusionMatrix doParseInstance(XContentParser parser) throws IOException {
         return MulticlassConfusionMatrix.fromXContent(parser);
@@ -78,12 +81,12 @@ public void testConstructor_SizeValidationFailures() {
 
     public void testAggs() {
         MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix();
-        Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs = confusionMatrix.aggs("act", "pred");
+        Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> aggs = confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred");
         assertThat(aggs, isTuple(not(empty()), empty()));
         assertThat(confusionMatrix.getResult(), isEmpty());
     }
 
-    public void testEvaluate() {
+    public void testProcess() {
         Aggregations aggs = new Aggregations(List.of(
             mockTerms(
                 MulticlassConfusionMatrix.STEP_1_AGGREGATE_BY_ACTUAL_CLASS,
@@ -91,6 +94,7 @@ public void testEvaluate() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 0L),
+            mockCardinality(MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 2L),
             mockFilters(
                 MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -105,13 +109,13 @@ public void testEvaluate() {
                         70,
                         new Aggregations(List.of(mockFilters(
                             MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L)))))))),
-            mockCardinality(MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 2L)));
+                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 0L))))))))
+        ));
 
         MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix(2, null);
         confusionMatrix.process(aggs);
 
-        assertThat(confusionMatrix.aggs("act", "pred"), isTuple(empty(), empty()));
+        assertThat(confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
         Result result = confusionMatrix.getResult().get();
         assertThat(result.getMetricName(), equalTo(MulticlassConfusionMatrix.NAME.getPreferredName()));
         assertThat(
@@ -123,7 +127,7 @@ public void testEvaluate() {
         assertThat(result.getOtherActualClassCount(), equalTo(0L));
     }
 
-    public void testEvaluate_OtherClassesCountGreaterThanZero() {
+    public void testProcess_OtherClassesCountGreaterThanZero() {
         Aggregations aggs = new Aggregations(List.of(
             mockTerms(
                 MulticlassConfusionMatrix.STEP_1_AGGREGATE_BY_ACTUAL_CLASS,
@@ -131,6 +135,7 @@ public void testEvaluate_OtherClassesCountGreaterThanZero() {
                     mockTermsBucket("dog", new Aggregations(List.of())),
                     mockTermsBucket("cat", new Aggregations(List.of()))),
                 100L),
+            mockCardinality(MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 5L),
             mockFilters(
                 MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
                 List.of(
@@ -145,13 +150,13 @@ public void testEvaluate_OtherClassesCountGreaterThanZero() {
                         85,
                         new Aggregations(List.of(mockFilters(
                             MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
-                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 15L)))))))),
-            mockCardinality(MulticlassConfusionMatrix.STEP_2_CARDINALITY_OF_ACTUAL_CLASS, 5L)));
+                            List.of(mockFiltersBucket("cat", 30L), mockFiltersBucket("dog", 40L), mockFiltersBucket("_other_", 15L))))))))
+        ));
 
         MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix(2, null);
         confusionMatrix.process(aggs);
 
-        assertThat(confusionMatrix.aggs("act", "pred"), isTuple(empty(), empty()));
+        assertThat(confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
         Result result = confusionMatrix.getResult().get();
         assertThat(result.getMetricName(), equalTo(MulticlassConfusionMatrix.NAME.getPreferredName()));
         assertThat(
@@ -162,4 +167,106 @@ public void testEvaluate_OtherClassesCountGreaterThanZero() {
                     new ActualClass("cat", 85, List.of(new PredictedClass("cat", 30L), new PredictedClass("dog", 40L)), 15))));
         assertThat(result.getOtherActualClassCount(), equalTo(3L));
     }
+
+    public void testProcess_MoreThanTwoStepsNeeded() {
+        Aggregations aggsStep1 = new Aggregations(List.of(
+            mockTerms(
+                MulticlassConfusionMatrix.STEP_1_AGGREGATE_BY_ACTUAL_CLASS,
+                List.of(
+                    mockTermsBucket("ant", new Aggregations(List.of())),
+                    mockTermsBucket("cat", new Aggregations(List.of())),
+                    mockTermsBucket("dog", new Aggregations(List.of())),
+                    mockTermsBucket("fox", new Aggregations(List.of()))),
+                0L),
+            mockCardinality(MulticlassConfusionMatrix.STEP_1_CARDINALITY_OF_ACTUAL_CLASS, 2L)
+        ));
+        Aggregations aggsStep2 = new Aggregations(List.of(
+            mockFilters(
+                MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
+                List.of(
+                    mockFiltersBucket(
+                        "ant",
+                        46,
+                        new Aggregations(List.of(mockFilters(
+                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
+                            List.of(
+                                mockFiltersBucket("ant", 10L),
+                                mockFiltersBucket("cat", 11L),
+                                mockFiltersBucket("dog", 12L),
+                                mockFiltersBucket("fox", 13L),
+                                mockFiltersBucket("_other_", 0L)))))),
+                    mockFiltersBucket(
+                        "cat",
+                        86,
+                        new Aggregations(List.of(mockFilters(
+                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
+                            List.of(
+                                mockFiltersBucket("ant", 20L),
+                                mockFiltersBucket("cat", 21L),
+                                mockFiltersBucket("dog", 22L),
+                                mockFiltersBucket("fox", 23L),
+                                mockFiltersBucket("_other_", 0L))))))))
+        ));
+        Aggregations aggsStep3 = new Aggregations(List.of(
+            mockFilters(
+                MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_ACTUAL_CLASS,
+                List.of(
+                    mockFiltersBucket(
+                        "dog",
+                        126,
+                        new Aggregations(List.of(mockFilters(
+                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
+                            List.of(
+                                mockFiltersBucket("ant", 30L),
+                                mockFiltersBucket("cat", 31L),
+                                mockFiltersBucket("dog", 32L),
+                                mockFiltersBucket("fox", 33L),
+                                mockFiltersBucket("_other_", 0L)))))),
+                    mockFiltersBucket(
+                        "fox",
+                        166,
+                        new Aggregations(List.of(mockFilters(
+                            MulticlassConfusionMatrix.STEP_2_AGGREGATE_BY_PREDICTED_CLASS,
+                            List.of(
+                                mockFiltersBucket("ant", 40L),
+                                mockFiltersBucket("cat", 41L),
+                                mockFiltersBucket("dog", 42L),
+                                mockFiltersBucket("fox", 43L),
+                                mockFiltersBucket("_other_", 0L))))))))
+        ));
+
+        MulticlassConfusionMatrix confusionMatrix = new MulticlassConfusionMatrix(4, null);
+        confusionMatrix.process(aggsStep1);
+        confusionMatrix.process(aggsStep2);
+        confusionMatrix.process(aggsStep3);
+
+        assertThat(confusionMatrix.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
+        Result result = confusionMatrix.getResult().get();
+        assertThat(result.getMetricName(), equalTo(MulticlassConfusionMatrix.NAME.getPreferredName()));
+        assertThat(
+            result.getConfusionMatrix(),
+            equalTo(
+                List.of(
+                    new ActualClass("ant", 46, List.of(
+                        new PredictedClass("ant", 10L),
+                        new PredictedClass("cat", 11L),
+                        new PredictedClass("dog", 12L),
+                        new PredictedClass("fox", 13L)), 0),
+                    new ActualClass("cat", 86, List.of(
+                        new PredictedClass("ant", 20L),
+                        new PredictedClass("cat", 21L),
+                        new PredictedClass("dog", 22L),
+                        new PredictedClass("fox", 23L)), 0),
+                    new ActualClass("dog", 126, List.of(
+                        new PredictedClass("ant", 30L),
+                        new PredictedClass("cat", 31L),
+                        new PredictedClass("dog", 32L),
+                        new PredictedClass("fox", 33L)), 0),
+                    new ActualClass("fox", 166, List.of(
+                        new PredictedClass("ant", 40L),
+                        new PredictedClass("cat", 41L),
+                        new PredictedClass("dog", 42L),
+                        new PredictedClass("fox", 43L)), 0))));
+        assertThat(result.getOtherActualClassCount(), equalTo(0L));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
new file mode 100644
index 0000000000000..5a397bae366d7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification;
+
+import org.elasticsearch.script.Script;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class PainlessScriptsTests extends ESTestCase {
+
+    public void testBuildIsEqualScript() {
+        Script script = PainlessScripts.buildIsEqualScript("act", "pred");
+        assertThat(script.getIdOrCode(), equalTo("String.valueOf(doc['act'].value).equals(String.valueOf(doc['pred'].value))"));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
index 362b81e9ae981..5e49412e3238b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -17,16 +18,18 @@
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty;
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockFilters;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockSingleValue;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
 public class PrecisionTests extends AbstractSerializingTestCase<Precision> {
 
+    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
+
     @Override
     protected Precision doParseInstance(XContentParser parser) throws IOException {
         return Precision.fromXContent(parser);
@@ -62,7 +65,7 @@ public void testProcess() {
         Precision precision = new Precision();
         precision.process(aggs);
 
-        assertThat(precision.aggs("act", "pred"), isTuple(empty(), empty()));
+        assertThat(precision.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
         assertThat(precision.getResult().get(), equalTo(new Precision.Result(List.of(), 0.8123)));
     }
 
@@ -112,7 +115,7 @@ public void testProcess_GivenCardinalityTooHigh() {
         Aggregations aggs =
             new Aggregations(Collections.singletonList(mockTerms(Precision.ACTUAL_CLASSES_NAMES_AGG_NAME, Collections.emptyList(), 1)));
         Precision precision = new Precision();
-        precision.aggs("foo", "bar");
+        precision.aggs(EVALUATION_PARAMETERS, "foo", "bar");
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> precision.process(aggs));
         assertThat(e.getMessage(), containsString("Cardinality of field [foo] is too high"));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
index be7d30411c931..721942203f779 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -17,15 +18,17 @@
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty;
+import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockSingleValue;
 import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MockAggregations.mockTerms;
-import static org.elasticsearch.test.hamcrest.TupleMatchers.isTuple;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
 public class RecallTests extends AbstractSerializingTestCase<Recall> {
 
+    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
+
     @Override
     protected Recall doParseInstance(XContentParser parser) throws IOException {
         return Recall.fromXContent(parser);
@@ -60,7 +63,7 @@ public void testProcess() {
         Recall recall = new Recall();
         recall.process(aggs);
 
-        assertThat(recall.aggs("act", "pred"), isTuple(empty(), empty()));
+        assertThat(recall.aggs(EVALUATION_PARAMETERS, "act", "pred"), isTuple(empty(), empty()));
         assertThat(recall.getResult().get(), equalTo(new Recall.Result(List.of(), 0.8123)));
     }
 
@@ -111,7 +114,7 @@ public void testProcess_GivenCardinalityTooHigh() {
             mockTerms(Recall.BY_ACTUAL_CLASS_AGG_NAME, Collections.emptyList(), 1),
             mockSingleValue(Recall.AVG_RECALL_AGG_NAME, 0.8123)));
         Recall recall = new Recall();
-        recall.aggs("foo", "bar");
+        recall.aggs(EVALUATION_PARAMETERS, "foo", "bar");
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> recall.process(aggs));
         assertThat(e.getMessage(), containsString("Cardinality of field [foo] is too high"));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
index 96ba97ecc9348..26dff097b1b32 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 
 import java.io.IOException;
@@ -28,6 +29,8 @@
 
 public class RegressionTests extends AbstractSerializingTestCase<Regression> {
 
+    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(MlEvaluationNamedXContentProvider.getNamedWriteables());
@@ -85,7 +88,7 @@ public void testBuildSearch() {
 
         Regression evaluation = new Regression("act", "pred", Arrays.asList(new MeanSquaredError()));
 
-        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(userProvidedQuery);
+        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(EVALUATION_PARAMETERS, userProvidedQuery);
         assertThat(searchSourceBuilder.query(), equalTo(expectedSearchQuery));
         assertThat(searchSourceBuilder.aggregations().count(), greaterThan(0));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
index 28e0a045b190d..2a9645e094291 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 
 import java.io.IOException;
@@ -28,6 +29,8 @@
 
 public class BinarySoftClassificationTests extends AbstractSerializingTestCase<BinarySoftClassification> {
 
+    private static final EvaluationParameters EVALUATION_PARAMETERS = new EvaluationParameters(100);
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(MlEvaluationNamedXContentProvider.getNamedWriteables());
@@ -98,7 +101,7 @@ public void testBuildSearch() {
 
         BinarySoftClassification evaluation = new BinarySoftClassification("act", "prob", Arrays.asList(new Precision(Arrays.asList(0.7))));
 
-        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(userProvidedQuery);
+        SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(EVALUATION_PARAMETERS, userProvidedQuery);
         assertThat(searchSourceBuilder.query(), equalTo(expectedSearchQuery));
         assertThat(searchSourceBuilder.aggregations().count(), greaterThan(0));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
index 11e66e69c0f75..a41f0564fb967 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
@@ -34,9 +34,11 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
 import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_INTERNAL_STORAGE;
@@ -137,7 +139,11 @@ public void testToXContentWithParams() throws IOException {
             TrainedModelInputTests.createRandomInput(),
             randomNonNegativeLong(),
             randomNonNegativeLong(),
-            "platinum");
+            "platinum",
+            randomBoolean() ? null :
+                Stream.generate(() -> randomAlphaOfLength(10))
+                    .limit(randomIntBetween(1, 10))
+                    .collect(Collectors.toMap(Function.identity(), (k) -> randomAlphaOfLength(10))));
 
         BytesReference reference = XContentHelper.toXContent(config, XContentType.JSON, ToXContent.EMPTY_PARAMS, false);
         assertThat(reference.utf8ToString(), containsString("\"compressed_definition\""));
@@ -172,7 +178,11 @@ public void testParseWithBothDefinitionAndCompressedSupplied() throws IOExceptio
             TrainedModelInputTests.createRandomInput(),
             randomNonNegativeLong(),
             randomNonNegativeLong(),
-            "platinum");
+            "platinum",
+            randomBoolean() ? null :
+                Stream.generate(() -> randomAlphaOfLength(10))
+                    .limit(randomIntBetween(1, 10))
+                    .collect(Collectors.toMap(Function.identity(), (k) -> randomAlphaOfLength(10))));
 
         BytesReference reference = XContentHelper.toXContent(config, XContentType.JSON, ToXContent.EMPTY_PARAMS, false);
         Map<String, Object> objectMap = XContentHelper.convertToMap(reference, true, XContentType.JSON).v2();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
index 80a55394a4003..850d197aa6d8b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
@@ -47,6 +47,7 @@
 
 import static java.util.stream.Collectors.toMap;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doAnswer;
@@ -67,7 +68,7 @@ public class MlIndexAndAliasTests extends ESTestCase {
     private IndicesAdminClient indicesAdminClient;
     private AdminClient adminClient;
     private Client client;
-    private ActionListener<Boolean> finalListener;
+    private ActionListener<Boolean> listener;
 
     private ArgumentCaptor<CreateIndexRequest> createRequestCaptor;
     private ArgumentCaptor<IndicesAliasesRequest> aliasesRequestCaptor;
@@ -91,7 +92,7 @@ public void setUpMocks() {
         when(client.threadPool()).thenReturn(threadPool);
         when(client.admin()).thenReturn(adminClient);
 
-        finalListener = mock(ActionListener.class);
+        listener = mock(ActionListener.class);
 
         createRequestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
         aliasesRequestCaptor = ArgumentCaptor.forClass(IndicesAliasesRequest.class);
@@ -99,17 +100,17 @@ public void setUpMocks() {
 
     @After
     public void verifyNoMoreInteractionsWithMocks() {
-        verifyNoMoreInteractions(indicesAdminClient, finalListener);
+        verifyNoMoreInteractions(indicesAdminClient, listener);
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_CleanState() {
         ClusterState clusterState = createClusterState(Collections.emptyMap());
         createIndexAndAliasIfNecessary(clusterState);
 
-        InOrder inOrder = inOrder(indicesAdminClient, finalListener);
+        InOrder inOrder = inOrder(indicesAdminClient, listener);
         inOrder.verify(indicesAdminClient).prepareCreate(FIRST_CONCRETE_INDEX);
         inOrder.verify(indicesAdminClient).create(createRequestCaptor.capture(), any());
-        inOrder.verify(finalListener).onResponse(true);
+        inOrder.verify(listener).onResponse(true);
 
         CreateIndexRequest createRequest = createRequestCaptor.getValue();
         assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
@@ -120,11 +121,7 @@ private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexN
         ClusterState clusterState = createClusterState(Collections.singletonMap(indexName, createIndexMetaDataWithAlias(indexName)));
         createIndexAndAliasIfNecessary(clusterState);
 
-        verify(finalListener).onResponse(false);
-    }
-
-    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtLegacyIndex() {
-        assertNoClientInteractionsWhenWriteAliasAlreadyExists(LEGACY_INDEX_WITHOUT_SUFFIX);
+        verify(listener).onResponse(false);
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtInitialStateIndex() {
@@ -132,23 +129,48 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPo
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtSubsequentStateIndex() {
-        assertNoClientInteractionsWhenWriteAliasAlreadyExists(".ml-state-000007");
+        assertNoClientInteractionsWhenWriteAliasAlreadyExists("test-000007");
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtDummyIndex() {
         assertNoClientInteractionsWhenWriteAliasAlreadyExists("dummy-index");
     }
 
+    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtLegacyStateIndex() {
+        ClusterState clusterState =
+            createClusterState(
+                Collections.singletonMap(LEGACY_INDEX_WITHOUT_SUFFIX, createIndexMetaDataWithAlias(LEGACY_INDEX_WITHOUT_SUFFIX)));
+        createIndexAndAliasIfNecessary(clusterState);
+
+        InOrder inOrder = inOrder(indicesAdminClient, listener);
+        inOrder.verify(indicesAdminClient).prepareCreate(FIRST_CONCRETE_INDEX);
+        inOrder.verify(indicesAdminClient).create(createRequestCaptor.capture(), any());
+        inOrder.verify(indicesAdminClient).prepareAliases();
+        inOrder.verify(indicesAdminClient).aliases(aliasesRequestCaptor.capture(), any());
+        inOrder.verify(listener).onResponse(true);
+
+        CreateIndexRequest createRequest = createRequestCaptor.getValue();
+        assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
+        assertThat(createRequest.aliases(), empty());
+
+        IndicesAliasesRequest indicesAliasesRequest = aliasesRequestCaptor.getValue();
+        assertThat(
+            indicesAliasesRequest.getAliasActions(),
+            contains(
+                AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX),
+                AliasActions.remove().alias(TEST_INDEX_ALIAS).index(LEGACY_INDEX_WITHOUT_SUFFIX)));
+    }
+
     private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> existingIndexNames, String expectedWriteIndexName) {
         ClusterState clusterState =
             createClusterState(
                 existingIndexNames.stream().collect(toMap(Function.identity(), MlIndexAndAliasTests::createIndexMetaData)));
         createIndexAndAliasIfNecessary(clusterState);
 
-        InOrder inOrder = inOrder(indicesAdminClient, finalListener);
+        InOrder inOrder = inOrder(indicesAdminClient, listener);
         inOrder.verify(indicesAdminClient).prepareAliases();
         inOrder.verify(indicesAdminClient).aliases(aliasesRequestCaptor.capture(), any());
-        inOrder.verify(finalListener).onResponse(true);
+        inOrder.verify(listener).onResponse(true);
 
         IndicesAliasesRequest indicesAliasesRequest = aliasesRequestCaptor.getValue();
         assertThat(
@@ -156,11 +178,6 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> e
             contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName)));
     }
 
-    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLegacyIndexExists() {
-        assertMlStateWriteAliasAddedToMostRecentMlStateIndex(
-            Arrays.asList(LEGACY_INDEX_WITHOUT_SUFFIX), LEGACY_INDEX_WITHOUT_SUFFIX);
-    }
-
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() {
         assertMlStateWriteAliasAddedToMostRecentMlStateIndex(
             Arrays.asList(FIRST_CONCRETE_INDEX), FIRST_CONCRETE_INDEX);
@@ -176,6 +193,21 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButBot
             Arrays.asList(LEGACY_INDEX_WITHOUT_SUFFIX, "test-000003", "test-000040", "test-000500"), "test-000500");
     }
 
+    public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLegacyStateIndexExists() {
+        ClusterState clusterState =
+            createClusterState(Collections.singletonMap(LEGACY_INDEX_WITHOUT_SUFFIX, createIndexMetaData(LEGACY_INDEX_WITHOUT_SUFFIX)));
+        createIndexAndAliasIfNecessary(clusterState);
+
+        InOrder inOrder = inOrder(indicesAdminClient, listener);
+        inOrder.verify(indicesAdminClient).prepareCreate(FIRST_CONCRETE_INDEX);
+        inOrder.verify(indicesAdminClient).create(createRequestCaptor.capture(), any());
+        inOrder.verify(listener).onResponse(true);
+
+        CreateIndexRequest createRequest = createRequestCaptor.getValue();
+        assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
+        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
+    }
+
     public void testIndexNameComparator() {
         Comparator<String> comparator = MlIndexAndAlias.INDEX_NAME_COMPARATOR;
         assertThat(
@@ -202,8 +234,8 @@ public void testIndexNameComparator() {
     }
 
     private void createIndexAndAliasIfNecessary(ClusterState clusterState) {
-        MlIndexAndAlias.createIndexAndAliasIfNecessary(client, clusterState, new IndexNameExpressionResolver(),
-            TEST_INDEX_PREFIX, TEST_INDEX_ALIAS, finalListener);
+        MlIndexAndAlias.createIndexAndAliasIfNecessary(
+            client, clusterState, new IndexNameExpressionResolver(), TEST_INDEX_PREFIX, TEST_INDEX_ALIAS, listener);
     }
 
     @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
new file mode 100644
index 0000000000000..c5087622404ab
--- /dev/null
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.deprecation;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+
+import java.util.Locale;
+
+public class NodeDeprecationChecks {
+
+    static DeprecationIssue checkRemovedSetting(final Settings settings, final Setting<?> removedSetting, final String url) {
+        if (removedSetting.exists(settings) == false) {
+            return null;
+        }
+        final String removedSettingKey = removedSetting.getKey();
+        final String value = removedSetting.get(settings).toString();
+        final String message =
+            String.format(Locale.ROOT, "setting [%s] is deprecated and will be removed in the next major version", removedSettingKey);
+        final String details =
+            String.format(Locale.ROOT, "the setting [%s] is currently set to [%s], remove this setting", removedSettingKey, value);
+        return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details);
+    }
+
+}
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
new file mode 100644
index 0000000000000..00c49aedbe223
--- /dev/null
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.deprecation;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+public class NodeDeprecationChecksTests extends ESTestCase {
+
+    public void testRemovedSettingNotSet() {
+        final Settings settings = Settings.EMPTY;
+        final Setting<?> removedSetting = Setting.simpleString("node.removed_setting");
+        final DeprecationIssue issue =
+            NodeDeprecationChecks.checkRemovedSetting(settings, removedSetting, "http://removed-setting.example.com");
+        assertThat(issue, nullValue());
+    }
+
+    public void testRemovedSetting() {
+        final Settings settings = Settings.builder().put("node.removed_setting", "value").build();
+        final Setting<?> removedSetting = Setting.simpleString("node.removed_setting");
+        final DeprecationIssue issue =
+            NodeDeprecationChecks.checkRemovedSetting(settings, removedSetting, "https://removed-setting.example.com");
+        assertThat(issue, not(nullValue()));
+        assertThat(issue.getLevel(), equalTo(DeprecationIssue.Level.CRITICAL));
+        assertThat(
+            issue.getMessage(),
+            equalTo("setting [node.removed_setting] is deprecated and will be removed in the next major version"));
+        assertThat(
+            issue.getDetails(),
+            equalTo("the setting [node.removed_setting] is currently set to [value], remove this setting"));
+        assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com"));
+    }
+
+}
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java
index f857232c7e910..1b6306fcb2a5c 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java
@@ -16,53 +16,28 @@
 import org.junit.Before;
 import org.junit.BeforeClass;
 
-import java.util.ArrayList;
-
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
 public abstract class CommonEqlRestTestCase extends ESRestTestCase {
 
-    static class SearchTestConfiguration {
-        final String input;
-        final int expectedStatus;
-        final String expectedMessage;
-
-        SearchTestConfiguration(String input, int status, String msg) {
-            this.input = input;
-            this.expectedStatus = status;
-            this.expectedMessage = msg;
-        }
-    }
-
-    public static final String defaultValidationIndexName = "eql_search_validation_test";
+    private static final String defaultValidationIndexName = "eql_search_validation_test";
     private static final String validQuery = "process where user = 'SYSTEM'";
 
-    public static final ArrayList<SearchTestConfiguration> searchValidationTests;
-    static {
-        searchValidationTests = new ArrayList<>();
-        searchValidationTests.add(new SearchTestConfiguration(null, 400, "request body or source parameter is required"));
-        searchValidationTests.add(new SearchTestConfiguration("{}", 400, "query is null or empty"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"\"}", 400, "query is null or empty"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"timestamp_field\": \"\"}",
-            400, "timestamp field is null or empty"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"event_category_field\": \"\"}",
-            400, "event category field is null or empty"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"implicit_join_key_field\": \"\"}",
-            400, "implicit join key field is null or empty"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"size\": 0}",
-            400, "size must be greater than 0"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"size\": -1}",
-            400, "size must be greater than 0"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"search_after\": null}",
-            400, "search_after doesn't support values of type: VALUE_NULL"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"search_after\": []}",
-            400, "must contains at least one value"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"filter\": null}",
-            400, "filter doesn't support values of type: VALUE_NULL"));
-        searchValidationTests.add(new SearchTestConfiguration("{\"query\": \"" + validQuery + "\", \"filter\": {}}",
-            400, "query malformed, empty clause found"));
-    }
+    private static final String[][] testBadRequests = {
+            {null, "request body or source parameter is required"},
+            {"{}", "query is null or empty"},
+            {"{\"query\": \"\"}", "query is null or empty"},
+            {"{\"query\": \"" + validQuery + "\", \"timestamp_field\": \"\"}", "timestamp field is null or empty"},
+            {"{\"query\": \"" + validQuery + "\", \"event_category_field\": \"\"}", "event category field is null or empty"},
+            {"{\"query\": \"" + validQuery + "\", \"implicit_join_key_field\": \"\"}", "implicit join key field is null or empty"},
+            {"{\"query\": \"" + validQuery + "\", \"size\": 0}", "size must be greater than 0"},
+            {"{\"query\": \"" + validQuery + "\", \"size\": -1}", "size must be greater than 0"},
+            {"{\"query\": \"" + validQuery + "\", \"search_after\": null}", "search_after doesn't support values of type: VALUE_NULL"},
+            {"{\"query\": \"" + validQuery + "\", \"search_after\": []}", "must contains at least one value"},
+            {"{\"query\": \"" + validQuery + "\", \"filter\": null}", "filter doesn't support values of type: VALUE_NULL"},
+            {"{\"query\": \"" + validQuery + "\", \"filter\": {}}", "query malformed, empty clause found"}
+    };
 
     @BeforeClass
     public static void checkForSnapshot() {
@@ -79,24 +54,19 @@ public void cleanup() throws Exception {
         deleteIndex(defaultValidationIndexName);
     }
 
-    public void testSearchValidationFailures() throws Exception {
+    public void testBadRequests() throws Exception {
         final String contentType = "application/json";
-        for (SearchTestConfiguration config : searchValidationTests) {
+        for (String[] test : testBadRequests) {
             final String endpoint = "/" + defaultValidationIndexName + "/_eql/search";
             Request request = new Request("GET", endpoint);
-            request.setJsonEntity(config.input);
+            request.setJsonEntity(test[0]);
 
-            Response response = null;
-            if (config.expectedStatus == 400) {
-                ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
-                response = e.getResponse();
-            } else {
-                response = client().performRequest(request);
-            }
+            ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
+            Response response = e.getResponse();
 
             assertThat(response.getHeader("Content-Type"), containsString(contentType));
-            assertThat(EntityUtils.toString(response.getEntity()), containsString(config.expectedMessage));
-            assertThat(response.getStatusLine().getStatusCode(), is(config.expectedStatus));
+            assertThat(EntityUtils.toString(response.getEntity()), containsString(test[1]));
+            assertThat(response.getStatusLine().getStatusCode(), is(400));
         }
     }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
index 979c622380072..aea8042c07ec0 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
@@ -8,6 +8,8 @@
 
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
@@ -43,6 +45,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
                 new BooleanLiteralsOnTheRight(),
                 // needs to occur before BinaryComparison combinations
                 new ReplaceWildcards(),
+                new ReplaceNullChecks(),
                 new PropagateEquals(),
                 new CombineBinaryComparisons(),
                 // prune/elimination
@@ -102,4 +105,28 @@ protected LogicalPlan rule(Filter filter) {
             });
         }
     }
+
+    private static class ReplaceNullChecks extends OptimizerRule<Filter> {
+
+        @Override
+        protected LogicalPlan rule(Filter filter) {
+
+            return filter.transformExpressionsUp(e -> {
+                // expr == null || expr != null
+                if (e instanceof Equals || e instanceof NotEquals) {
+                    BinaryComparison cmp = (BinaryComparison) e;
+
+                    if (cmp.right().foldable() && cmp.right().fold() == null) {
+                        if (e instanceof Equals) {
+                            e = new IsNull(e.source(), cmp.left());
+                        } else {
+                            e = new IsNotNull(e.source(), cmp.left());
+                        }
+                    }
+                }
+
+                return e;
+            });
+        }
+    }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
index 17f9f0da8c6f9..cf313acd1a1b6 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
@@ -7,33 +7,32 @@
 package org.elasticsearch.xpack.eql.action;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.elasticsearch.Build;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.SearchHit;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.util.ArrayList;
-import java.util.Iterator;
+import java.util.HashMap;
 import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class EqlActionIT extends AbstractEqlIntegTestCase {
 
     static final String indexPrefix = "endgame";
     static final String testIndexName = indexPrefix + "-1.4.0";
-    protected static final String PARAM_FORMATTING = "%1$s.test";
-
+    protected static final String PARAM_FORMATTING = "%1$s.test -> %2$s";
 
     @BeforeClass
     public static void checkForSnapshot() {
@@ -41,26 +40,29 @@ public static void checkForSnapshot() {
     }
 
     @Before
+    @SuppressWarnings("unchecked")
     public void setUpData() throws Exception {
         // Insert test data
         ObjectMapper mapper = new ObjectMapper();
         BulkRequestBuilder bulkBuilder = client().prepareBulk();
-        JsonNode rootNode = mapper.readTree(EqlActionIT.class.getResourceAsStream("/test_data.json"));
-        Iterator<JsonNode> entries = rootNode.elements();
-        while (entries.hasNext()) {
-            JsonNode entry = entries.next();
-
-            // Adjust the structure of the document with additional event.category and @timestamp fields
-            // Add event.category field
-            ObjectNode objEvent = ((ObjectNode)entry).putObject("event");
-            JsonNode objEventType = entry.get("event_type");
-            objEvent.put("category", objEventType.asText());
-
-            // Add @timestamp field
-            JsonNode objTimestamp = entry.get("timestamp");
-            ((ObjectNode)entry).put("@timestamp", objTimestamp.asLong());
-
-            bulkBuilder.add(new IndexRequest(testIndexName).source(entry.toString(), XContentType.JSON));
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, EqlActionIT.class.getResourceAsStream("/test_data.json"))) {
+            List<Object> list = parser.list();
+            for (Object item : list) {
+                assertThat(item, instanceOf(HashMap.class));
+
+                HashMap<String, Object> entry = (HashMap<String, Object>) item;
+
+                // Adjust the structure of the document with additional event.category and @timestamp fields
+                // Add event.category field
+                HashMap<String, Object> objEvent = new HashMap<>();
+                objEvent.put("category", entry.get("event_type"));
+                entry.put("event", objEvent);
+
+                // Add @timestamp field
+                entry.put("@timestamp", entry.get("timestamp"));
+
+                bulkBuilder.add(new IndexRequest(testIndexName).source(entry, XContentType.JSON));
+            }
         }
         BulkResponse bulkResponse = bulkBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
         assertThat(bulkResponse.hasFailures() ? bulkResponse.buildFailureMessage() : "", bulkResponse.hasFailures(), equalTo(false));
@@ -108,19 +110,24 @@ public EqlActionIT(int num, EqlSpec spec) {
         this.spec = spec;
     }
 
-    public final void test() {
-        EqlSearchResponse response = new EqlSearchRequestBuilder(client(), EqlSearchAction.INSTANCE)
-            .indices(testIndexName).query(spec.query()).get();
-
-        List<SearchHit> events = response.hits().events();
-        assertNotNull(events);
-
+    private static long[] extractIds(List<SearchHit> events) {
         final int len = events.size();
         final long ids[] = new long[len];
-        for (int i = 0; i < events.size(); i++) {
+        for (int i = 0; i < len; i++) {
             ids[i] = ((Number) events.get(i).getSourceAsMap().get("serial_event_id")).longValue();
         }
-        final String msg = "unexpected result for spec: [" + spec.toString() + "]";
-        assertArrayEquals(msg, spec.expectedEventIds(), ids);
+        return ids;
+    }
+
+    private void assertSpec(List<SearchHit> events) {
+        assertNotNull(events);
+        assertArrayEquals("unexpected result for spec: [" + spec.toString() + "]", spec.expectedEventIds(), extractIds(events));
+    }
+
+    public final void test() {
+        EqlSearchResponse response = new EqlSearchRequestBuilder(client(), EqlSearchAction.INSTANCE)
+                .indices(testIndexName).query(spec.query()).get();
+
+        assertSpec(response.hits().events());
     }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
index 6b53fec8caafb..bbd589331b8ad 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
@@ -24,6 +26,8 @@
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.ql.type.TypesTests;
 
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 
 public class OptimizerTests extends ESTestCase {
@@ -32,6 +36,7 @@ public class OptimizerTests extends ESTestCase {
     private static final String INDEX_NAME = "test";
     private EqlParser parser = new EqlParser();
     private IndexResolution index = loadIndexResolution("mapping-default.json");
+
     private static Map<String, EsField> loadEqlMapping(String name) {
         return TypesTests.loadMapping(name);
     }
@@ -51,9 +56,54 @@ private LogicalPlan accept(String eql) {
         return accept(index, eql);
     }
 
+    public void testIsNull() {
+        List<String> tests = Arrays.asList(
+            "foo where command_line == null",
+            "foo where null == command_line"
+        );
+
+        for (String q : tests) {
+            LogicalPlan plan = accept(q);
+            assertTrue(plan instanceof OrderBy);
+            plan = ((OrderBy) plan).child();
+            assertTrue(plan instanceof Filter);
+
+            Filter filter = (Filter) plan;
+            And condition = (And) filter.condition();
+            assertTrue(condition.right() instanceof IsNull);
+
+            IsNull check = (IsNull) condition.right();
+            assertEquals(((FieldAttribute) check.field()).name(), "command_line");
+        }
+    }
+    public void testIsNotNull() {
+        List<String> tests = Arrays.asList(
+            "foo where command_line != null",
+            "foo where null != command_line"
+        );
+
+        for (String q : tests) {
+            LogicalPlan plan = accept(q);
+            assertTrue(plan instanceof OrderBy);
+            plan = ((OrderBy) plan).child();
+            assertTrue(plan instanceof Filter);
+
+            Filter filter = (Filter) plan;
+            And condition = (And) filter.condition();
+            assertTrue(condition.right() instanceof IsNotNull);
+
+            IsNotNull check = (IsNotNull) condition.right();
+            assertEquals(((FieldAttribute) check.field()).name(), "command_line");
+        }
+    }
 
     public void testEqualsWildcard() {
-        for (String q : new String[]{"foo where command_line == '* bar *'", "foo where '* bar *' == command_line"}) {
+        List<String> tests = Arrays.asList(
+            "foo where command_line == '* bar *'",
+            "foo where '* bar *' == command_line"
+        );
+
+        for (String q : tests) {
             LogicalPlan plan = accept(q);
             assertTrue(plan instanceof OrderBy);
             plan = ((OrderBy) plan).child();
@@ -72,7 +122,12 @@ public void testEqualsWildcard() {
     }
 
     public void testNotEqualsWildcard() {
-        for (String q : new String[]{"foo where command_line != '* baz *'", "foo where '* baz *' != command_line"}) {
+        List<String> tests = Arrays.asList(
+            "foo where command_line != '* baz *'",
+            "foo where '* baz *' != command_line"
+        );
+
+        for (String q : tests) {
 
             LogicalPlan plan = accept(q);
             assertTrue(plan instanceof OrderBy);
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java
new file mode 100644
index 0000000000000..7c54e31db79be
--- /dev/null
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.planner;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.eql.analysis.Analyzer;
+import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
+import org.elasticsearch.xpack.eql.analysis.Verifier;
+import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
+import org.elasticsearch.xpack.eql.optimizer.Optimizer;
+import org.elasticsearch.xpack.eql.parser.EqlParser;
+import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
+import org.elasticsearch.xpack.ql.index.EsIndex;
+import org.elasticsearch.xpack.ql.index.IndexResolution;
+
+import static org.elasticsearch.xpack.ql.type.TypesTests.loadMapping;
+
+public abstract class AbstractQueryFolderTestCase extends ESTestCase {
+    protected EqlParser parser = new EqlParser();
+    protected PreAnalyzer preAnalyzer = new PreAnalyzer();
+    protected Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier());
+    protected Optimizer optimizer = new Optimizer();
+    protected Planner planner = new Planner();
+
+    protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json")));
+
+    protected PhysicalPlan plan(IndexResolution resolution, String eql) {
+        return planner.plan(optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))));
+    }
+
+    protected PhysicalPlan plan(String eql) {
+        return plan(index, eql);
+    }
+}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
new file mode 100644
index 0000000000000..471ee4c56943a
--- /dev/null
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.planner;
+
+import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
+
+public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
+    public void testPropertyEquationFilterUnsupported() {
+        QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
+                () -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)"));
+        String msg = e.getMessage();
+        assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg);
+    }
+
+    public void testPropertyEquationInClauseFilterUnsupported() {
+        QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
+                () -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")"));
+        String msg = e.getMessage();
+        assertEquals("Line 1:52: Comparisons against variables are not (currently) supported; offender [parent_process_name] in [==]", msg);
+    }
+}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
new file mode 100644
index 0000000000000..acbe328a4740a
--- /dev/null
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.planner;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
+import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
+
+import java.util.Arrays;
+
+import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
+import static org.hamcrest.Matchers.containsString;
+
+public class QueryFolderOkTests extends AbstractQueryFolderTestCase {
+    private static Object[][] specs = {
+            {"basic", "process where true", null},
+            {"singleNumericFilterEquals", "process where serial_event_id = 1", "\"term\":{\"serial_event_id\":{\"value\":1"},
+            {"singleNumericFilterLess", "process where serial_event_id < 4",
+                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":false"
+            },
+            {"singleNumericFilterLessSymmetry", "process where 4 > serial_event_id",
+                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":false"
+            },
+            {"singleNumericFilterLessEquals", "process where serial_event_id <= 4",
+                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":true"
+            },
+            {"singleNumericFilterGreater", "process where serial_event_id > 4",
+                    "\"range\":{\"serial_event_id\":{\"from\":4,\"to\":null,\"include_lower\":false,\"include_upper\":false"
+            },
+            {"singleNumericFilterGreaterEquals", "process where serial_event_id >= 4",
+                    "\"range\":{\"serial_event_id\":{\"from\":4,\"to\":null,\"include_lower\":true,\"include_upper\":false"
+            },
+            {"mixedTypeFilter", "process where process_name == \"notepad.exe\" or (serial_event_id < 4.5 and serial_event_id >= 3.1)",
+                    new Object[]{
+                            "\"term\":{\"process_name\":{\"value\":\"notepad.exe\"",
+                            "\"range\":{\"serial_event_id\":{\"from\":3.1,\"to\":4.5,\"include_lower\":true,\"include_upper\":false"
+                    }
+            },
+            {"notFilter", "process where not (exit_code > -1)",
+                    "\"range\":{\"exit_code\":{\"from\":null,\"to\":-1,\"include_lower\":false,\"include_upper\":true"
+            },
+            {"inFilter", "process where process_name in (\"python.exe\", \"SMSS.exe\", \"explorer.exe\")",
+                    new Object[]{
+                            "\"term\":{\"process_name\":{\"value\":\"python.exe\"",
+                            "\"term\":{\"process_name\":{\"value\":\"SMSS.exe\"",
+                            "\"term\":{\"process_name\":{\"value\":\"explorer.exe\"",
+                    }
+            },
+            {"equalsAndInFilter", "process where process_path == \"*\\\\red_ttp\\\\wininit.*\" and opcode in (0,1,2,3)",
+                    new Object[]{
+                            "\"wildcard\":{\"process_path\":{\"wildcard\":\"*\\\\\\\\red_ttp\\\\\\\\wininit.*\"",
+                            "\"term\":{\"opcode\":{\"value\":0",
+                            "\"term\":{\"opcode\":{\"value\":1",
+                            "\"term\":{\"opcode\":{\"value\":2",
+                            "\"term\":{\"opcode\":{\"value\":3",
+                    }
+            },
+    };
+
+    private final String name;
+    private final String query;
+    private final Object expect;
+
+    public QueryFolderOkTests(String name, String query, Object expect) {
+        this.name = name;
+        this.query = query;
+        this.expect = expect;
+    }
+
+    @ParametersFactory(shuffle = false, argumentFormatting = "%1$s.test")
+    public static Iterable<Object[]> parameters() {
+        return Arrays.asList(specs);
+    }
+
+    public void test() {
+        PhysicalPlan p = plan(query);
+        assertEquals(EsQueryExec.class, p.getClass());
+        EsQueryExec eqe = (EsQueryExec) p;
+        assertEquals(23, eqe.output().size());
+        assertEquals(KEYWORD, eqe.output().get(0).dataType());
+
+        final String query = eqe.queryContainer().toString().replaceAll("\\s+", "");
+
+        // test query term
+        if (expect != null) {
+            if (expect instanceof Object[]) {
+                for (Object item : (Object[]) expect) {
+                    assertThat(query, containsString((String) item));
+                }
+            } else {
+                assertThat(query, containsString((String) expect));
+            }
+        }
+
+        // test common term
+        assertThat(query, containsString("\"term\":{\"event.category\":{\"value\":\"process\""));
+
+        // test field source extraction
+        assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]"));
+    }
+}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java
deleted file mode 100644
index 1c0664ee7ac99..0000000000000
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.eql.planner;
-
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.eql.analysis.Analyzer;
-import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
-import org.elasticsearch.xpack.eql.analysis.Verifier;
-import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
-import org.elasticsearch.xpack.eql.optimizer.Optimizer;
-import org.elasticsearch.xpack.eql.parser.EqlParser;
-import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
-import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.ql.index.EsIndex;
-import org.elasticsearch.xpack.ql.index.IndexResolution;
-
-import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.ql.type.TypesTests.loadMapping;
-import static org.hamcrest.Matchers.containsString;
-
-public class QueryFolderTests extends ESTestCase {
-
-    private EqlParser parser = new EqlParser();
-    private PreAnalyzer preAnalyzer = new PreAnalyzer();
-    private Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier());
-    private Optimizer optimizer = new Optimizer();
-    private Planner planner = new Planner();
-
-    private IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json")));
-
-
-    private PhysicalPlan plan(IndexResolution resolution, String eql) {
-        return planner.plan(optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))));
-    }
-
-    private PhysicalPlan plan(String eql) {
-        return plan(index, eql);
-    }
-    
-    public void testBasicPlan() {
-        PhysicalPlan p = plan("process where true");
-        assertEquals(EsQueryExec.class, p.getClass());
-        EsQueryExec eqe = (EsQueryExec) p;
-        assertEquals(23, eqe.output().size());
-        assertEquals(KEYWORD, eqe.output().get(0).dataType());
-        String query = eqe.queryContainer().toString().replaceAll("\\s+", "");
-        // test query term
-        assertThat(query, containsString("\"term\":{\"event.category\":{\"value\":\"process\""));
-        // test field source extraction
-        assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]"));
-    }
-}
diff --git a/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml
index f8a96eaef12aa..b510f80d37f9f 100644
--- a/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml
+++ b/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml
@@ -12,14 +12,11 @@
 # query = 'process where serial_event_id = 1'
 # expected_event_ids  = [1]
 
-[[queries]]
-query = 'process where serial_event_id < 4'
-expected_event_ids  = [1, 2, 3]
-
 [[queries]]
 query = 'process where true | head 6'
 expected_event_ids  = [1, 2, 3, 4, 5, 6]
 
+# presently not supported, throwing: org.elasticsearch.xpack.ql.rule.RuleExecutionException: Does not know how to handle a local relation
 [[queries]]
 query = 'process where false'
 expected_event_ids = []
@@ -32,12 +29,6 @@ query = 'process where missing_field != null'
 expected_event_ids  = [1, 2, 3, 4, 5]
 query = 'process where bad_field == null | head 5'
 
-[[queries]]
-query = '''
-  process where process_name == "impossible name" or (serial_event_id < 4.5 and serial_event_id >= 3.1)
-'''
-expected_event_ids  = [4]
-
 [[queries]]
 tags = ["comparisons", "pipes"]
 query = '''
@@ -70,42 +61,6 @@ process where true
 '''
 expected_event_ids  = [9, 10]
 
-[[queries]]
-query = '''
-process where serial_event_id<=8 and serial_event_id > 7
-'''
-expected_event_ids  = [8]
-
-[[queries]]
-note = "check that comparisons against null values return false"
-expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
-query = 'process where exit_code >= 0'
-
-[[queries]]
-note = "check that comparisons against null values return false"
-expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
-query = 'process where 0 <= exit_code'
-
-[[queries]]
-note = "check that comparisons against null values return false"
-expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
-query = 'process where exit_code <= 0'
-
-[[queries]]
-note = "check that comparisons against null values return false"
-expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
-query = 'process where exit_code < 1'
-
-[[queries]]
-note = "check that comparisons against null values return false"
-expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
-query = 'process where exit_code > -1'
-
-[[queries]]
-note = "check that comparisons against null values return false"
-expected_event_ids  = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303]
-query = 'process where -1 < exit_code'
-
 [[queries]]
 note = "check that comparisons against null values return false"
 expected_event_ids = []
@@ -125,26 +80,6 @@ note = "check that comparisons against null values return false"
 expected_event_ids  = [1, 2, 3, 4, 5, 6, 7]
 query = 'process where not (-1 < exit_code) | head 7'
 
-[[queries]]
-query = 'process where exit_code > 0'
-expected_event_ids = []
-
-[[queries]]
-query = 'process where exit_code < 0'
-expected_event_ids = []
-
-[[queries]]
-query = 'process where 0 < exit_code'
-expected_event_ids = []
-
-[[queries]]
-query = 'process where 0 > exit_code'
-expected_event_ids = []
-
-[[queries]]
-query = 'process where (serial_event_id<=8 and serial_event_id > 7) and (opcode=3 and opcode>2)'
-expected_event_ids  = [8]
-
 [[queries]]
 query = 'process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)'
 expected_event_ids  = [7, 8]
@@ -217,16 +152,6 @@ expected_event_ids  = [57]
 query = '''
 registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] == "EN"'''
 
-[[queries]]
-query = '''
-registry where key_path == "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"'''
-expected_event_ids  = [79]
-
-[[queries]]
-query = '''
-process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)'''
-expected_event_ids  = [84, 85]
-
 [[queries]]
 query = '''
 file where file_name == "csrss.exe" and opcode=0
@@ -253,34 +178,6 @@ process where opcode=1 and process_name == "smss.exe"
 '''
 expected_event_ids  = [78]
 
-[[queries]]
-query = '''
-file where file_path="*\\red_ttp\\winin*.*"
-  and opcode in (0,1,2) and user_name="vagrant"
-'''
-expected_event_ids  = [83, 86]
-
-[[queries]]
-query = '''
-file where file_path="*\\red_ttp\\winin*.*"
-  and opcode not in (0,1,2) and user_name="vagrant"
-'''
-expected_event_ids  = []
-
-[[queries]]
-query = '''
-file where file_path="*\\red_ttp\\winin*.*"
-  and opcode not in (3, 4, 5, 6 ,7) and user_name="vagrant"
-'''
-expected_event_ids  = [83, 86]
-
-
-[[queries]]
-query = '''
-file where file_name in ("wininit.exe", "lsass.exe") and opcode == 2
-'''
-expected_event_ids  = [65, 86]
-
 [[queries]]
 query = '''
 file where true
@@ -612,15 +509,6 @@ query = '''
 registry where length(bad_field) > 0
 '''
 
-[[queries]]
-query = '''
-process where opcode == 1
-  and process_name in ("net.exe", "net1.exe")
-  and not (parent_process_name == "net.exe"
-  and process_name == "net1.exe")
-  and command_line == "*group *admin*" and command_line != "* /add*"'''
-expected_event_ids  = [97]
-
 [[queries]]
 expected_event_ids  = [1, 55, 57, 63, 75304]
 query = '''
@@ -690,11 +578,6 @@ file where event of [process where process_name = "python.exe" ]
 | unique unique_pid'''
 expected_event_ids  = [55, 95]
 
-[[queries]]
-query = '''
-process where process_name = "python.exe"'''
-expected_event_ids  = [48, 50, 51, 54, 93]
-
 [[queries]]
 query = 'process where event of [process where process_name = "python.exe" ]'
 expected_event_ids  = [48, 50, 51, 54, 93]
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index 5a22857494557..d043117412e56 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
@@ -69,7 +70,8 @@
 public final class FrozenEngine extends ReadOnlyEngine {
     public static final Setting<Boolean> INDEX_FROZEN = Setting.boolSetting("index.frozen", false, Setting.Property.IndexScope,
         Setting.Property.PrivateIndex);
-    private final SegmentsStats stats;
+    private final SegmentsStats segmentsStats;
+    private final DocsStats docsStats;
     private volatile ElasticsearchDirectoryReader lastOpenedReader;
     private final ElasticsearchDirectoryReader canMatchReader;
 
@@ -78,15 +80,15 @@ public FrozenEngine(EngineConfig config) {
 
         boolean success = false;
         Directory directory = store.directory();
-        // Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
-        try (DirectoryReader reader = openDirectory(directory, false)) {
-            // we record the segment stats here - that's what the reader needs when it's open and it give the user
+        try (DirectoryReader reader = openDirectory(directory)) {
+            // we record the segment stats and doc stats here - that's what the reader needs when it's open and it give the user
             // an idea of what it can save when it's closed
-            this.stats = new SegmentsStats();
+            this.segmentsStats = new SegmentsStats();
             for (LeafReaderContext ctx : reader.getContext().leaves()) {
                 SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
-                fillSegmentStats(segmentReader, true, stats);
+                fillSegmentStats(segmentReader, true, segmentsStats);
             }
+            this.docsStats = docsStats(reader);
             final DirectoryReader wrappedReader = new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
             canMatchReader = ElasticsearchDirectoryReader.wrap(
                 new RewriteCachingDirectoryReader(directory, wrappedReader.leaves()), config.getShardId());
@@ -170,7 +172,7 @@ private synchronized ElasticsearchDirectoryReader getOrOpenReader() throws IOExc
                 for (ReferenceManager.RefreshListener listeners : config ().getInternalRefreshListener()) {
                     listeners.beforeRefresh();
                 }
-                final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory(), true);
+                final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory());
                 reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
                 processReader(reader);
                 reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);
@@ -594,7 +596,7 @@ public LeafReader getDelegate() {
     public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
         if (includeUnloadedSegments) {
             final SegmentsStats stats = new SegmentsStats();
-            stats.add(this.stats);
+            stats.add(this.segmentsStats);
             if (includeSegmentFileSizes == false) {
                 stats.clearFileSizes();
             }
@@ -605,6 +607,11 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
 
     }
 
+    @Override
+    public DocsStats docStats() {
+        return docsStats;
+    }
+
     synchronized boolean isReaderOpen() {
         return lastOpenedReader != null;
     } // this is mainly for tests
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 00414b20679e3..3c53b78e4523b 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.xpack.core.ilm.AllocateAction;
 import org.elasticsearch.xpack.core.ilm.DeleteAction;
 import org.elasticsearch.xpack.core.ilm.DeleteStep;
-import org.elasticsearch.xpack.core.ilm.ErrorStep;
 import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
 import org.elasticsearch.xpack.core.ilm.FreezeAction;
 import org.elasticsearch.xpack.core.ilm.FreezeStep;
@@ -1396,6 +1395,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception {
         assertBusy(() -> assertHistoryIsPresent(policy, index + "-000002", true, "check-rollover-ready"), 30, TimeUnit.SECONDS);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50353")
     public void testHistoryIsWrittenWithFailure() throws Exception {
         createIndexWithSettings(index + "-1", Settings.builder(), false);
         createNewSingletonPolicy("hot", new RolloverAction(null, null, 1L));
@@ -1406,7 +1406,8 @@ public void testHistoryIsWrittenWithFailure() throws Exception {
         Request refreshIndex = new Request("POST", "/" + index + "-1/_refresh");
         client().performRequest(refreshIndex);
 
-        assertBusy(() -> assertThat(getStepKeyForIndex(index + "-1").getName(), equalTo(ErrorStep.NAME)), 30, TimeUnit.SECONDS);
+        // Check that we've had error and auto retried
+        assertBusy(() -> assertThat((Integer) explainIndex(index + "-1").get("failed_step_retry_count"), greaterThanOrEqualTo(1)));
 
         assertBusy(() -> assertHistoryIsPresent(policy, index + "-1", false, "ERROR"), 30, TimeUnit.SECONDS);
     }
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
index de77dbed9f7f4..f2c9e9c609502 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
@@ -91,6 +91,7 @@ public void testMissingRepo() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50358")
     public void testFullPolicySnapshot() throws Exception {
         final String indexName = "test";
         final String policyName = "full-policy";
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index 4f4c2913fb2a4..965c5beb5af04 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -100,7 +100,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
-        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
 
         // This is necessary to prevent ILM and SLM installing a lifecycle policy, these tests assume a blank slate
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
index 180ff53cf5b97..58f0b632e1321 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
@@ -50,7 +50,6 @@ protected Settings nodeSettings() {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
-        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         settings.put(Environment.PATH_REPO_SETTING.getKey(), repositoryLocation);
 
         settings.put(XPackSettings.SNAPSHOT_LIFECYCLE_ENABLED.getKey(), true);
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
index 6523d7a07228a..b3149cc177e5a 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.slm;
 
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -30,6 +31,18 @@ public class SnapshotLifecyclePolicyTests extends AbstractSerializingTestCase<Sn
 
     private String id;
 
+    public void testToRequest() {
+        SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy("id", "name", "0 1 2 3 4 ? 2099", "repo", Collections.emptyMap(),
+            SnapshotRetentionConfiguration.EMPTY);
+        CreateSnapshotRequest request = p.toRequest();
+        CreateSnapshotRequest expected = new CreateSnapshotRequest().userMetadata(Collections.singletonMap("policy", "id"));
+
+        p = new SnapshotLifecyclePolicy("id", "name", "0 1 2 3 4 ? 2099", "repo", null, null);
+        request = p.toRequest();
+        expected.waitForCompletion(true).snapshot(request.snapshot()).repository("repo");
+        assertEquals(expected, request);
+    }
+
     public void testNameGeneration() {
         long time = 1552684146542L; // Fri Mar 15 2019 21:09:06 UTC
         SnapshotLifecyclePolicy.ResolverContext context = new SnapshotLifecyclePolicy.ResolverContext(time);
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
index 177135974fb6c..3d22402e89779 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
-import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
@@ -37,24 +36,27 @@ public class Logstash extends Plugin implements SystemIndexPlugin {
     private static final String OLD_LOGSTASH_INDEX_NAME = "logstash-index-template";
     private static final String TEMPLATE_VERSION_VARIABLE = "logstash.template.version";
 
-    private final boolean enabled;
-
-    public Logstash(Settings settings) {
-        this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings);
-    }
+    public Logstash() {}
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
             new ActionHandler<>(XPackUsageFeatureAction.LOGSTASH, LogstashUsageTransportAction.class),
-            new ActionHandler<>(XPackInfoFeatureAction.LOGSTASH, LogstashInfoTransportAction.class));
+            new ActionHandler<>(XPackInfoFeatureAction.LOGSTASH, LogstashInfoTransportAction.class)
+        );
     }
 
     public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
         return templates -> {
             templates.keySet().removeIf(OLD_LOGSTASH_INDEX_NAME::equals);
-            TemplateUtils.loadTemplateIntoMap("/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json", templates, LOGSTASH_INDEX_TEMPLATE_NAME,
-                    Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE, LogManager.getLogger(Logstash.class));
+            TemplateUtils.loadTemplateIntoMap(
+                "/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json",
+                templates,
+                LOGSTASH_INDEX_TEMPLATE_NAME,
+                Version.CURRENT.toString(),
+                TEMPLATE_VERSION_VARIABLE,
+                LogManager.getLogger(Logstash.class)
+            );
             assert templates.get(LOGSTASH_INDEX_TEMPLATE_NAME).mappings() != null;
             return templates;
         };
@@ -62,7 +64,8 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return Collections.singletonList(new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME,
-            "Contains data for Logstash Central Management"));
+        return Collections.singletonList(
+            new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME, "Contains data for Logstash Central Management")
+        );
     }
 }
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
index 49a48daf5ea9c..4aa77af2988f4 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
@@ -21,8 +21,12 @@ public class LogstashInfoTransportAction extends XPackInfoFeatureTransportAction
     private final XPackLicenseState licenseState;
 
     @Inject
-    public LogstashInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
-                                       Settings settings, XPackLicenseState licenseState) {
+    public LogstashInfoTransportAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        Settings settings,
+        XPackLicenseState licenseState
+    ) {
         super(XPackInfoFeatureAction.LOGSTASH.name(), transportService, actionFilters);
         this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings);
         this.licenseState = licenseState;
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
index 55f096ec2e8e9..e03cc822ec746 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
@@ -29,21 +29,36 @@ public class LogstashUsageTransportAction extends XPackUsageFeatureTransportActi
     private final XPackLicenseState licenseState;
 
     @Inject
-    public LogstashUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
-                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                        Settings settings, XPackLicenseState licenseState) {
-        super(XPackUsageFeatureAction.LOGSTASH.name(), transportService, clusterService,
-            threadPool, actionFilters, indexNameExpressionResolver);
+    public LogstashUsageTransportAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Settings settings,
+        XPackLicenseState licenseState
+    ) {
+        super(
+            XPackUsageFeatureAction.LOGSTASH.name(),
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            indexNameExpressionResolver
+        );
         this.settings = settings;
         this.licenseState = licenseState;
     }
 
     @Override
-    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
-                                   ActionListener<XPackUsageFeatureResponse> listener) {
+    protected void masterOperation(
+        Task task,
+        XPackUsageRequest request,
+        ClusterState state,
+        ActionListener<XPackUsageFeatureResponse> listener
+    ) {
         boolean available = licenseState.isLogstashAllowed();
-        LogstashFeatureSetUsage usage =
-            new LogstashFeatureSetUsage(available, XPackSettings.LOGSTASH_ENABLED.get(settings));
+        LogstashFeatureSetUsage usage = new LogstashFeatureSetUsage(available, XPackSettings.LOGSTASH_ENABLED.get(settings));
         listener.onResponse(new XPackUsageFeatureResponse(usage));
     }
 }
diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
index 42115fbf74c07..e3872352b944d 100644
--- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
+++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
@@ -8,11 +8,13 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
 
@@ -24,12 +26,13 @@ public class LogstashInfoTransportActionTests extends ESTestCase {
 
     public void testEnabledSetting() throws Exception {
         boolean enabled = randomBoolean();
-        Settings settings = Settings.builder()
-            .put("path.home", createTempDir())
-            .put("xpack.logstash.enabled", enabled)
-            .build();
+        Settings settings = Settings.builder().put("path.home", createTempDir()).put("xpack.logstash.enabled", enabled).build();
         LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
-            mock(TransportService.class), mock(ActionFilters.class), settings, null);
+            mock(TransportService.class),
+            mock(ActionFilters.class),
+            settings,
+            null
+        );
         assertThat(featureSet.enabled(), is(enabled));
 
         LogstashUsageTransportAction usageAction = newUsageAction(settings, false);
@@ -41,19 +44,29 @@ public void testEnabledSetting() throws Exception {
         usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new LogstashFeatureSetUsage(out.bytes().streamInput());
         assertThat(serializedUsage.enabled(), is(enabled));
+
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] { XPackSettings.LOGSTASH_ENABLED });
     }
 
     public void testEnabledDefault() throws Exception {
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
         LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
-            mock(TransportService.class), mock(ActionFilters.class), settings, null);
+            mock(TransportService.class),
+            mock(ActionFilters.class),
+            settings,
+            null
+        );
         assertThat(featureSet.enabled(), is(true));
     }
 
     public void testAvailable() throws Exception {
         final XPackLicenseState licenseState = mock(XPackLicenseState.class);
         LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
-            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
+            mock(TransportService.class),
+            mock(ActionFilters.class),
+            Settings.EMPTY,
+            licenseState
+        );
         boolean available = randomBoolean();
         when(licenseState.isLogstashAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -73,7 +86,14 @@ public void testAvailable() throws Exception {
     private LogstashUsageTransportAction newUsageAction(Settings settings, boolean available) {
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isLogstashAllowed()).thenReturn(available);
-        return new LogstashUsageTransportAction(mock(TransportService.class), null,
-            null, mock(ActionFilters.class), null, settings, licenseState);
+        return new LogstashUsageTransportAction(
+            mock(TransportService.class),
+            null,
+            null,
+            mock(ActionFilters.class),
+            null,
+            settings,
+            licenseState
+        );
     }
 }
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index d0383b0999e8d..e4702bb0245c0 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -50,6 +50,7 @@ dependencies {
   compileOnly project(':modules:lang-painless:spi')
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('ilm'), configuration: 'default')
   // This should not be here
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
 
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index 2ac378e472a7b..516e5c626fca2 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -76,8 +76,8 @@ integTest.runner {
     'ml/data_frame_analytics_crud/Test put regression given negative gamma',
     'ml/data_frame_analytics_crud/Test put regression given eta less than 1e-3',
     'ml/data_frame_analytics_crud/Test put regression given eta greater than one',
-    'ml/data_frame_analytics_crud/Test put regression given maximum_number_trees is zero',
-    'ml/data_frame_analytics_crud/Test put regression given maximum_number_trees is greater than 2k',
+    'ml/data_frame_analytics_crud/Test put regression given max_trees is zero',
+    'ml/data_frame_analytics_crud/Test put regression given max_trees is greater than 2k',
     'ml/data_frame_analytics_crud/Test put regression given feature_bag_fraction is negative',
     'ml/data_frame_analytics_crud/Test put regression given feature_bag_fraction is greater than one',
     'ml/data_frame_analytics_crud/Test put regression given training_percent is less than one',
@@ -87,8 +87,8 @@ integTest.runner {
     'ml/data_frame_analytics_crud/Test put classification given negative gamma',
     'ml/data_frame_analytics_crud/Test put classification given eta less than 1e-3',
     'ml/data_frame_analytics_crud/Test put classification given eta greater than one',
-    'ml/data_frame_analytics_crud/Test put classification given maximum_number_trees is zero',
-    'ml/data_frame_analytics_crud/Test put classification given maximum_number_trees is greater than 2k',
+    'ml/data_frame_analytics_crud/Test put classification given max_trees is zero',
+    'ml/data_frame_analytics_crud/Test put classification given max_trees is greater than 2k',
     'ml/data_frame_analytics_crud/Test put classification given feature_bag_fraction is negative',
     'ml/data_frame_analytics_crud/Test put classification given feature_bag_fraction is greater than one',
     'ml/data_frame_analytics_crud/Test put classification given num_top_classes is less than zero',
@@ -139,6 +139,9 @@ integTest.runner {
     'ml/inference_crud/Test put ensemble with empty models',
     'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index',
     'ml/inference_crud/Test put model with empty input.field_names',
+    'ml/inference_processor/Test create processor with missing mandatory fields',
+    'ml/inference_processor/Test create and delete pipeline with inference processor',
+    'ml/inference_processor/Test create processor with deprecated fields',
     'ml/inference_stats_crud/Test get stats given missing trained model',
     'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false',
     'ml/jobs_crud/Test cannot create job with existing categorizer state document',
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
index 4ca8fb5a64373..04f71e0c6f7d4 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -80,7 +79,7 @@ public void setUpData() {
     public void tearDownData() {
         cleanUp();
         client().admin().indices().prepareDelete(DATA_INDEX).get();
-        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
+        refresh("*");
     }
 
     public void testBasicCategorization() throws Exception {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
index ccf39aa8aa472..773f91b5027d2 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
@@ -5,18 +5,21 @@
  */
 package org.elasticsearch.xpack.ml.integration;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService.TooManyBucketsException;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Classification;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall;
 import org.junit.After;
 import org.junit.Before;
 
@@ -27,16 +30,20 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 
 public class ClassificationEvaluationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
 
     private static final String ANIMALS_DATA_INDEX = "test-evaluate-animals-index";
 
-    private static final String ANIMAL_NAME_FIELD = "animal_name";
+    private static final String ANIMAL_NAME_KEYWORD_FIELD = "animal_name_keyword";
     private static final String ANIMAL_NAME_PREDICTION_FIELD = "animal_name_prediction";
-    private static final String NO_LEGS_FIELD = "no_legs";
+    private static final String NO_LEGS_KEYWORD_FIELD = "no_legs_keyword";
+    private static final String NO_LEGS_INTEGER_FIELD = "no_legs_integer";
     private static final String NO_LEGS_PREDICTION_FIELD = "no_legs_prediction";
-    private static final String IS_PREDATOR_FIELD = "predator";
+    private static final String IS_PREDATOR_KEYWORD_FIELD = "predator_keyword";
+    private static final String IS_PREDATOR_BOOLEAN_FIELD = "predator_boolean";
     private static final String IS_PREDATOR_PREDICTION_FIELD = "predator_prediction";
 
     @Before
@@ -48,11 +55,15 @@ public void setup() {
     @After
     public void cleanup() {
         cleanUp();
+        client().admin().cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().putNull("search.max_buckets"))
+            .get();
     }
 
     public void testEvaluate_DefaultMetrics() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, null));
+            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, null));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(
@@ -65,7 +76,7 @@ public void testEvaluate_AllMetrics() {
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
                 new Classification(
-                    ANIMAL_NAME_FIELD,
+                    ANIMAL_NAME_KEYWORD_FIELD,
                     ANIMAL_NAME_PREDICTION_FIELD,
                     List.of(new Accuracy(), new MulticlassConfusionMatrix(), new Precision(), new Recall())));
 
@@ -82,7 +93,7 @@ public void testEvaluate_AllMetrics() {
     public void testEvaluate_Accuracy_KeywordField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Accuracy())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -101,10 +112,9 @@ public void testEvaluate_Accuracy_KeywordField() {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(5.0 / 75));
     }
 
-    public void testEvaluate_Accuracy_IntegerField() {
+    private void evaluateAccuracy_IntegerField(String actualField) {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(NO_LEGS_FIELD, NO_LEGS_PREDICTION_FIELD, List.of(new Accuracy())));
+            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -123,10 +133,18 @@ public void testEvaluate_Accuracy_IntegerField() {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(15.0 / 75));
     }
 
-    public void testEvaluate_Accuracy_BooleanField() {
+    public void testEvaluate_Accuracy_IntegerField() {
+        evaluateAccuracy_IntegerField(NO_LEGS_INTEGER_FIELD);
+    }
+
+    public void testEvaluate_Accuracy_IntegerField_MappingTypeMismatch() {
+        evaluateAccuracy_IntegerField(NO_LEGS_KEYWORD_FIELD);
+    }
+
+    private void evaluateAccuracy_BooleanField(String actualField) {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(IS_PREDATOR_FIELD, IS_PREDATOR_PREDICTION_FIELD, List.of(new Accuracy())));
+                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -142,10 +160,18 @@ public void testEvaluate_Accuracy_BooleanField() {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(45.0 / 75));
     }
 
-    public void testEvaluate_Precision() {
+    public void testEvaluate_Accuracy_BooleanField() {
+        evaluateAccuracy_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
+    }
+
+    public void testEvaluate_Accuracy_BooleanField_MappingTypeMismatch() {
+        evaluateAccuracy_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
+    }
+
+    public void testEvaluate_Precision_KeywordField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -164,20 +190,78 @@ public void testEvaluate_Precision() {
         assertThat(precisionResult.getAvgPrecision(), equalTo(5.0 / 75));
     }
 
+    private void evaluatePrecision_IntegerField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_PREDICTION_FIELD, List.of(new Precision())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Precision.Result precisionResult = (Precision.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(precisionResult.getMetricName(), equalTo(Precision.NAME.getPreferredName()));
+        assertThat(
+            precisionResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Precision.PerClassResult("1", 0.2),
+                    new Precision.PerClassResult("2", 0.2),
+                    new Precision.PerClassResult("3", 0.2),
+                    new Precision.PerClassResult("4", 0.2),
+                    new Precision.PerClassResult("5", 0.2))));
+        assertThat(precisionResult.getAvgPrecision(), equalTo(0.2));
+    }
+
+    public void testEvaluate_Precision_IntegerField() {
+        evaluatePrecision_IntegerField(NO_LEGS_INTEGER_FIELD);
+    }
+
+    public void testEvaluate_Precision_IntegerField_MappingTypeMismatch() {
+        evaluatePrecision_IntegerField(NO_LEGS_KEYWORD_FIELD);
+    }
+
+    private void evaluatePrecision_BooleanField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Precision())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Precision.Result precisionResult = (Precision.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(precisionResult.getMetricName(), equalTo(Precision.NAME.getPreferredName()));
+        assertThat(
+            precisionResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Precision.PerClassResult("false", 0.5),
+                    new Precision.PerClassResult("true", 9.0 / 13))));
+        assertThat(precisionResult.getAvgPrecision(), equalTo(31.0 / 52));
+    }
+
+    public void testEvaluate_Precision_BooleanField() {
+        evaluatePrecision_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
+    }
+
+    public void testEvaluate_Precision_BooleanField_MappingTypeMismatch() {
+        evaluatePrecision_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
+    }
+
     public void testEvaluate_Precision_CardinalityTooHigh() {
         indexDistinctAnimals(ANIMALS_DATA_INDEX, 1001);
         ElasticsearchStatusException e =
             expectThrows(
                 ElasticsearchStatusException.class,
                 () -> evaluateDataFrame(
-                    ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision()))));
-        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name] is too high"));
+                    ANIMALS_DATA_INDEX,
+                    new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision()))));
+        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name_keyword] is too high"));
     }
 
-    public void testEvaluate_Recall() {
+    public void testEvaluate_Recall_KeywordField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -196,21 +280,79 @@ public void testEvaluate_Recall() {
         assertThat(recallResult.getAvgRecall(), equalTo(5.0 / 75));
     }
 
+    private void evaluateRecall_IntegerField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_INTEGER_FIELD, List.of(new Recall())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(recallResult.getMetricName(), equalTo(Recall.NAME.getPreferredName()));
+        assertThat(
+            recallResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Recall.PerClassResult("1", 1.0),
+                    new Recall.PerClassResult("2", 1.0),
+                    new Recall.PerClassResult("3", 1.0),
+                    new Recall.PerClassResult("4", 1.0),
+                    new Recall.PerClassResult("5", 1.0))));
+        assertThat(recallResult.getAvgRecall(), equalTo(1.0));
+    }
+
+    public void testEvaluate_Recall_IntegerField() {
+        evaluateRecall_IntegerField(NO_LEGS_INTEGER_FIELD);
+    }
+
+    public void testEvaluate_Recall_IntegerField_MappingTypeMismatch() {
+        evaluateRecall_IntegerField(NO_LEGS_KEYWORD_FIELD);
+    }
+
+    private void evaluateRecall_BooleanField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Recall())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(recallResult.getMetricName(), equalTo(Recall.NAME.getPreferredName()));
+        assertThat(
+            recallResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Recall.PerClassResult("true", 0.6),
+                    new Recall.PerClassResult("false", 0.6))));
+        assertThat(recallResult.getAvgRecall(), equalTo(0.6));
+    }
+
+    public void testEvaluate_Recall_BooleanField() {
+        evaluateRecall_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
+    }
+
+    public void testEvaluate_Recall_BooleanField_MappingTypeMismatch() {
+        evaluateRecall_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
+    }
+
     public void testEvaluate_Recall_CardinalityTooHigh() {
         indexDistinctAnimals(ANIMALS_DATA_INDEX, 1001);
         ElasticsearchStatusException e =
             expectThrows(
                 ElasticsearchStatusException.class,
                 () -> evaluateDataFrame(
-                    ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall()))));
-        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name] is too high"));
+                    ANIMALS_DATA_INDEX,
+                    new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall()))));
+        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name_keyword] is too high"));
     }
 
-    public void testEvaluate_ConfusionMatrixMetricWithDefaultSize() {
+    private void evaluateWithMulticlassConfusionMatrix() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
-                new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix())));
+                new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -269,11 +411,29 @@ public void testEvaluate_ConfusionMatrixMetricWithDefaultSize() {
         assertThat(confusionMatrixResult.getOtherActualClassCount(), equalTo(0L));
     }
 
+    public void testEvaluate_ConfusionMatrixMetricWithDefaultSize() {
+        evaluateWithMulticlassConfusionMatrix();
+
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("search.max_buckets", 20)).get();
+        evaluateWithMulticlassConfusionMatrix();
+
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("search.max_buckets", 7)).get();
+        evaluateWithMulticlassConfusionMatrix();
+
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("search.max_buckets", 6)).get();
+        ElasticsearchException e = expectThrows(ElasticsearchException.class, this::evaluateWithMulticlassConfusionMatrix);
+
+        assertThat(e.getCause(), is(instanceOf(TooManyBucketsException.class)));
+        TooManyBucketsException tmbe = (TooManyBucketsException) e.getCause();
+        assertThat(tmbe.getMaxBuckets(), equalTo(6));
+    }
+
     public void testEvaluate_ConfusionMatrixMetricWithUserProvidedSize() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
-                new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix(3, null))));
+                new Classification(
+                    ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix(3, null))));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -310,11 +470,13 @@ public void testEvaluate_ConfusionMatrixMetricWithUserProvidedSize() {
     private static void createAnimalsIndex(String indexName) {
         client().admin().indices().prepareCreate(indexName)
             .setMapping(
-                ANIMAL_NAME_FIELD, "type=keyword",
+                ANIMAL_NAME_KEYWORD_FIELD, "type=keyword",
                 ANIMAL_NAME_PREDICTION_FIELD, "type=keyword",
-                NO_LEGS_FIELD, "type=integer",
+                NO_LEGS_KEYWORD_FIELD, "type=keyword",
+                NO_LEGS_INTEGER_FIELD, "type=integer",
                 NO_LEGS_PREDICTION_FIELD, "type=integer",
-                IS_PREDATOR_FIELD, "type=boolean",
+                IS_PREDATOR_KEYWORD_FIELD, "type=keyword",
+                IS_PREDATOR_BOOLEAN_FIELD, "type=boolean",
                 IS_PREDATOR_PREDICTION_FIELD, "type=boolean")
             .get();
     }
@@ -329,11 +491,13 @@ private static void indexAnimalsData(String indexName) {
                     bulkRequestBuilder.add(
                         new IndexRequest(indexName)
                             .source(
-                                ANIMAL_NAME_FIELD, animalNames.get(i),
+                                ANIMAL_NAME_KEYWORD_FIELD, animalNames.get(i),
                                 ANIMAL_NAME_PREDICTION_FIELD, animalNames.get((i + j) % animalNames.size()),
-                                NO_LEGS_FIELD, i + 1,
+                                NO_LEGS_KEYWORD_FIELD, String.valueOf(i + 1),
+                                NO_LEGS_INTEGER_FIELD, i + 1,
                                 NO_LEGS_PREDICTION_FIELD, j + 1,
-                                IS_PREDATOR_FIELD, i % 2 == 0,
+                                IS_PREDATOR_KEYWORD_FIELD, String.valueOf(i % 2 == 0),
+                                IS_PREDATOR_BOOLEAN_FIELD, i % 2 == 0,
                                 IS_PREDATOR_PREDICTION_FIELD, (i + j) % 2 == 0));
                 }
             }
@@ -349,7 +513,8 @@ private static void indexDistinctAnimals(String indexName, int distinctAnimalCou
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         for (int i = 0; i < distinctAnimalCount; i++) {
             bulkRequestBuilder.add(
-                new IndexRequest(indexName).source(ANIMAL_NAME_FIELD, "animal_" + i, ANIMAL_NAME_PREDICTION_FIELD, randomAlphaOfLength(5)));
+                new IndexRequest(indexName)
+                    .source(ANIMAL_NAME_KEYWORD_FIELD, "animal_" + i, ANIMAL_NAME_PREDICTION_FIELD, randomAlphaOfLength(5)));
         }
         BulkResponse bulkResponse = bulkRequestBuilder.get();
         if (bulkResponse.hasFailures()) {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index 8c029c5f84bb2..a58418bbd35ef 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -32,7 +31,6 @@
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall;
 import org.junit.After;
-import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -72,25 +70,9 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
     private String destIndex;
     private boolean analysisUsesExistingDestIndex;
 
-    @Before
-    public void enableLogging() {
-        client().admin().cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder()
-                .put("logger.org.elasticsearch.xpack.ml.action", "DEBUG")
-                .put("logger.org.elasticsearch.xpack.ml.dataframe", "DEBUG"))
-            .get();
-    }
-
     @After
     public void cleanup() {
         cleanUp();
-        client().admin().cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder()
-                .putNull("logger.org.elasticsearch.xpack.ml.action")
-                .putNull("logger.org.elasticsearch.xpack.ml.dataframe"))
-            .get();
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
@@ -106,6 +88,7 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
                 null,
                 null,
                 null,
+                null,
                 null));
         registerAnalytics(config);
         putAnalytics(config);
@@ -207,7 +190,7 @@ public <T> void testWithOnlyTrainingRowsAndTrainingPercentIsFifty(String jobId,
                 sourceIndex,
                 destIndex,
                 null,
-                new Classification(dependentVariable, BoostedTreeParams.builder().build(), null, numTopClasses, 50.0, null));
+                new Classification(dependentVariable, BoostedTreeParams.builder().build(), null, null, numTopClasses, 50.0, null));
         registerAnalytics(config);
         putAnalytics(config);
 
@@ -339,9 +322,11 @@ public void testStopAndRestart() throws Exception {
         assertEvaluation(KEYWORD_FIELD, KEYWORD_FIELD_VALUES, "ml." + predictedClassField);
     }
 
+    @AwaitsFix(bugUrl = "Muted until ml-cpp supports multiple classes")
     public void testDependentVariableCardinalityTooHighError() throws Exception {
         initialize("cardinality_too_high");
         indexData(sourceIndex, 6, 5, KEYWORD_FIELD);
+
         // Index one more document with a class different than the two already used.
         client().execute(IndexAction.INSTANCE, new IndexRequest(sourceIndex)
             .source(KEYWORD_FIELD, "fox")
@@ -435,7 +420,6 @@ public void testDependentVariableIsAliasToNested() throws Exception {
         assertEvaluation(ALIAS_TO_NESTED_FIELD, KEYWORD_FIELD_VALUES, "ml." + predictedClassField);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53188")
     public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exception {
         String sourceIndex = "classification_two_jobs_with_same_randomize_seed_source";
         String dependentVariable = KEYWORD_FIELD;
@@ -453,11 +437,11 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
             .setGamma(1.0)
             .setEta(1.0)
             .setFeatureBagFraction(1.0)
-            .setMaximumNumberTrees(1)
+            .setMaxTrees(1)
             .build();
 
         DataFrameAnalyticsConfig firstJob = buildAnalytics(firstJobId, sourceIndex, firstJobDestIndex, null,
-            new Classification(dependentVariable, boostedTreeParams, null, 1, 50.0, null));
+            new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, null));
         registerAnalytics(firstJob);
         putAnalytics(firstJob);
 
@@ -466,7 +450,7 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
 
         long randomizeSeed = ((Classification) firstJob.getAnalysis()).getRandomizeSeed();
         DataFrameAnalyticsConfig secondJob = buildAnalytics(secondJobId, sourceIndex, secondJobDestIndex, null,
-            new Classification(dependentVariable, boostedTreeParams, null, 1, 50.0, randomizeSeed));
+            new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, randomizeSeed));
 
         registerAnalytics(secondJob);
         putAnalytics(secondJob);
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
index 7d1506286362b..ca0c99eb4b421 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -27,6 +26,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.Detector;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
+import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats;
@@ -92,12 +92,14 @@ public void testDeleteExpiredData_GivenNothingToDelete() throws Exception {
 
     public void testDeleteExpiredData() throws Exception {
         // Index some unused state documents (more than 10K to test scrolling works)
-        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        String mlStateIndexName = AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-000001";
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         for (int i = 0; i < 10010; i++) {
             String docId = "non_existing_job_" + randomFrom("model_state_1234567#" + i, "quantiles", "categorizer_state#" + i);
-            IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).id(docId);
-            indexRequest.source(Collections.emptyMap());
+            IndexRequest indexRequest =
+                new IndexRequest(mlStateIndexName)
+                    .id(docId)
+                    .source(Collections.emptyMap());
             bulkRequestBuilder.add(indexRequest);
         }
         ActionFuture<BulkResponse> indexUnusedStateDocsResponse = bulkRequestBuilder.execute();
@@ -162,7 +164,7 @@ public void testDeleteExpiredData() throws Exception {
         }
 
         // Refresh to ensure the snapshot timestamp updates are visible
-        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
+        refresh("*");
 
         // We need to wait a second to ensure the second time around model snapshots will have a different ID (it depends on epoch seconds)
         // FIXME it would be better to wait for something concrete instead of wait for time to elapse
@@ -293,6 +295,6 @@ private void retainAllSnapshots(String jobId) throws Exception {
             client().execute(UpdateModelSnapshotAction.INSTANCE, request).get();
         }
         // We need to refresh to ensure the updates are visible
-        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
+        refresh("*");
     }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
index 422cb00cdb192..d6c88fbfdb994 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
@@ -120,7 +120,7 @@ public void testSimulate() throws IOException {
             "                \"num_top_feature_importance_values\": 2" +
             "          }},\n" +
             "          \"model_id\": \"test_classification\",\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -133,7 +133,7 @@ public void testSimulate() throws IOException {
             "          \"target_field\": \"ml.regression\",\n" +
             "          \"model_id\": \"test_regression\",\n" +
             "          \"inference_config\": {\"regression\":{}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -166,7 +166,7 @@ public void testSimulate() throws IOException {
             "        \"inference\": {\n" +
             "          \"model_id\": \"test_classification_missing\",\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -191,6 +191,40 @@ public void testSimulate() throws IOException {
         assertThat(responseString, containsString("Could not find trained model [test_classification_missing]"));
     }
 
+    public void testSimulateWithDefaultMappedField() throws IOException {
+        String source = "{\n" +
+            "  \"pipeline\": {\n" +
+            "    \"processors\": [\n" +
+            "      {\n" +
+            "        \"inference\": {\n" +
+            "          \"target_field\": \"ml.classification\",\n" +
+            "          \"inference_config\": {\"classification\": " +
+            "                {\"num_top_classes\":2, " +
+            "                \"top_classes_results_field\": \"result_class_prob\"," +
+            "                \"num_top_feature_importance_values\": 2" +
+            "          }},\n" +
+            "          \"model_id\": \"test_classification\",\n" +
+            "          \"field_map\": {}\n" +
+            "        }\n" +
+            "      }\n"+
+            "    ]\n" +
+            "  },\n" +
+            "  \"docs\": [\n" +
+            "    {\"_source\": {\n" +
+            "      \"col_1_alias\": \"female\",\n" +
+            "      \"col2\": \"M\",\n" +
+            "      \"col3\": \"none\",\n" +
+            "      \"col4\": 10\n" +
+            "    }}]\n" +
+            "}";
+
+        Response response = client().performRequest(simulateRequest(source));
+        String responseString = EntityUtils.toString(response.getEntity());
+        assertThat(responseString, containsString("\"predicted_value\":\"second\""));
+        assertThat(responseString, containsString("\"col2\":0.944"));
+        assertThat(responseString, containsString("\"col1\":0.19999"));
+    }
+
     public void testSimulateLangIdent() throws IOException {
         String source = "{\n" +
             "  \"pipeline\": {\n" +
@@ -199,7 +233,7 @@ public void testSimulateLangIdent() throws IOException {
             "        \"inference\": {\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
             "          \"model_id\": \"lang_ident_model_1\",\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }\n" +
             "    ]\n" +
@@ -525,6 +559,7 @@ protected NamedXContentRegistry xContentRegistry() {
         "{\n" +
         "  \"input\":{\"field_names\":[\"col1\",\"col2\",\"col3\",\"col4\"]}," +
         "  \"description\": \"test model for classification\",\n" +
+        "  \"default_field_map\": {\"col_1_alias\": \"col1\"},\n" +
         "  \"definition\": " + CLASSIFICATION_DEFINITION +
         "}";
 
@@ -535,7 +570,7 @@ protected NamedXContentRegistry xContentRegistry() {
         "          \"model_id\": \"test_classification\",\n" +
         "          \"tag\": \"classification\",\n" +
         "          \"inference_config\": {\"classification\": {}},\n" +
-        "          \"field_mappings\": {\n" +
+        "          \"field_map\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +
@@ -551,7 +586,7 @@ protected NamedXContentRegistry xContentRegistry() {
         "          \"model_id\": \"test_regression\",\n" +
         "          \"tag\": \"regression\",\n" +
         "          \"inference_config\": {\"regression\": {}},\n" +
-        "          \"field_mappings\": {\n" +
+        "          \"field_map\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
index d8cba1c772589..f91afb7ac6e34 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
@@ -28,6 +27,7 @@
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.transport.Netty4Plugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -40,6 +40,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 
 import java.io.IOException;
@@ -73,7 +74,12 @@ protected NamedXContentRegistry xContentRegistry() {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LocalStateMachineLearning.class, Netty4Plugin.class, ReindexPlugin.class);
+        return Arrays.asList(
+            LocalStateMachineLearning.class,
+            Netty4Plugin.class,
+            ReindexPlugin.class,
+            // ILM is required for .ml-state template index settings
+            IndexLifecycle.class);
     }
 
     @Override
@@ -115,6 +121,7 @@ protected Settings externalClusterClientSettings() {
         builder.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
+        builder.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         builder.put(Environment.PATH_HOME_SETTING.getKey(), home);
         builder.put("xpack.security.transport.ssl.enabled", true);
@@ -149,7 +156,7 @@ protected DeleteExpiredDataAction.Response deleteExpiredData() throws Exception
             new DeleteExpiredDataAction.Request()).get();
 
         // We need to refresh to ensure the deletion is visible
-        client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
+        refresh("*");
 
         return response;
     }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
index 92e0137462eee..5fc6d618da9ab 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
@@ -21,7 +20,6 @@
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
 import org.junit.After;
-import org.junit.Before;
 
 import java.util.List;
 import java.util.Map;
@@ -45,25 +43,9 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase {
     private String sourceIndex;
     private String destIndex;
 
-    @Before
-    public void enableLogging() {
-        client().admin().cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder()
-                .put("logger.org.elasticsearch.xpack.ml.action", "DEBUG")
-                .put("logger.org.elasticsearch.xpack.ml.dataframe", "DEBUG"))
-            .get();
-    }
-
     @After
     public void cleanup() {
         cleanUp();
-        client().admin().cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder()
-                .putNull("logger.org.elasticsearch.xpack.ml.action")
-                .putNull("logger.org.elasticsearch.xpack.ml.dataframe"))
-            .get();
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
@@ -287,7 +269,6 @@ public void testStopAndRestart() throws Exception {
         assertMlResultsFieldMappings(destIndex, predictedClassField, "double");
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53188")
     public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exception {
         String sourceIndex = "regression_two_jobs_with_same_randomize_seed_source";
         indexData(sourceIndex, 100, 0);
@@ -300,7 +281,7 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
             .setGamma(1.0)
             .setEta(1.0)
             .setFeatureBagFraction(1.0)
-            .setMaximumNumberTrees(1)
+            .setMaxTrees(1)
             .build();
 
         DataFrameAnalyticsConfig firstJob = buildAnalytics(firstJobId, sourceIndex, firstJobDestIndex, null,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index e77799ca1c0de..d235982be4344 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -73,6 +73,7 @@
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAction;
+import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
@@ -144,6 +145,7 @@
 import org.elasticsearch.xpack.ml.action.TransportDeleteJobAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteModelSnapshotAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteTrainedModelAction;
+import org.elasticsearch.xpack.ml.action.TransportEstimateModelMemoryAction;
 import org.elasticsearch.xpack.ml.action.TransportEvaluateDataFrameAction;
 import org.elasticsearch.xpack.ml.action.TransportExplainDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.ml.action.TransportFinalizeJobExecutionAction;
@@ -281,6 +283,7 @@
 import org.elasticsearch.xpack.ml.rest.job.RestCloseJobAction;
 import org.elasticsearch.xpack.ml.rest.job.RestDeleteForecastAction;
 import org.elasticsearch.xpack.ml.rest.job.RestDeleteJobAction;
+import org.elasticsearch.xpack.ml.rest.job.RestEstimateModelMemoryAction;
 import org.elasticsearch.xpack.ml.rest.job.RestFlushJobAction;
 import org.elasticsearch.xpack.ml.rest.job.RestForecastJobAction;
 import org.elasticsearch.xpack.ml.rest.job.RestGetJobStatsAction;
@@ -733,6 +736,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             new RestFlushJobAction(),
             new RestValidateDetectorAction(),
             new RestValidateJobConfigAction(),
+            new RestEstimateModelMemoryAction(),
             new RestGetCategoriesAction(),
             new RestGetModelSnapshotsAction(),
             new RestRevertModelSnapshotAction(),
@@ -811,6 +815,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                 new ActionHandler<>(FlushJobAction.INSTANCE, TransportFlushJobAction.class),
                 new ActionHandler<>(ValidateDetectorAction.INSTANCE, TransportValidateDetectorAction.class),
                 new ActionHandler<>(ValidateJobConfigAction.INSTANCE, TransportValidateJobConfigAction.class),
+                new ActionHandler<>(EstimateModelMemoryAction.INSTANCE, TransportEstimateModelMemoryAction.class),
                 new ActionHandler<>(GetCategoriesAction.INSTANCE, TransportGetCategoriesAction.class),
                 new ActionHandler<>(GetModelSnapshotsAction.INSTANCE, TransportGetModelSnapshotsAction.class),
                 new ActionHandler<>(RevertModelSnapshotAction.INSTANCE, TransportRevertModelSnapshotAction.class),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
index 4f731d9804c4b..7f36db7a5b893 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.ingest.IngestStats;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
@@ -40,6 +41,7 @@
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
@@ -113,6 +115,8 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat
                 addInferenceIngestUsage(response, inferenceUsage);
                 SearchRequestBuilder requestBuilder = client.prepareSearch(InferenceIndexConstants.INDEX_PATTERN)
                     .setSize(0)
+                    .setQuery(QueryBuilders.boolQuery()
+                        .filter(QueryBuilders.termQuery(InferenceIndexConstants.DOC_TYPE.getPreferredName(), TrainedModelConfig.NAME)))
                     .setTrackTotalHits(true);
                 ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(),
                     ClientHelper.ML_ORIGIN,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
index ae516b4f8ce7a..83c0050035d38 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.MlConfigIndex;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlStatsIndex;
@@ -38,10 +39,8 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
 
     private static final IndexTemplateConfig ANOMALY_DETECTION_RESULTS_TEMPLATE = anomalyDetectionResultsTemplate();
 
-    private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = new IndexTemplateConfig(
-        AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,ANOMALY_DETECTION_PATH + "state_index_template.json",
-        Version.CURRENT.id, VERSION_PATTERN,
-        Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
+    private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = stateTemplate(true);
+    private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE_NO_ILM = stateTemplate(false);
 
     private static final IndexTemplateConfig META_TEMPLATE = new IndexTemplateConfig(MlMetaIndex.INDEX_NAME,
         ROOT_RESOURCE_PATH + "meta_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
@@ -57,7 +56,12 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
         ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
         Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
 
-    private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate();
+    private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate(true);
+    private static final IndexTemplateConfig STATS_TEMPLATE_NO_ILM = statsTemplate(false);
+
+    private static final String ML_SIZE_BASED_ILM_POLICY_NAME = "ml-size-based-ilm-policy";
+    private static final LifecyclePolicyConfig ML_SIZE_BASED_ILM_POLICY =
+        new LifecyclePolicyConfig(ML_SIZE_BASED_ILM_POLICY_NAME, ROOT_RESOURCE_PATH + "size_based_ilm_policy.json");
 
     private static IndexTemplateConfig configTemplate() {
         Map<String, String> variables = new HashMap<>();
@@ -72,6 +76,22 @@ private static IndexTemplateConfig configTemplate() {
             variables);
     }
 
+    private static IndexTemplateConfig stateTemplate(boolean ilmEnabled) {
+        Map<String, String> variables = new HashMap<>();
+        variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
+        variables.put(
+            "xpack.ml.index.lifecycle.settings",
+            ilmEnabled
+                ? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" +
+                  ",\"index.lifecycle.rollover_alias\": \"" + AnomalyDetectorsIndex.jobStateIndexWriteAlias() + "\"\n"
+                : "");
+
+        return new IndexTemplateConfig(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
+            ANOMALY_DETECTION_PATH + "state_index_template.json",
+            Version.CURRENT.id, VERSION_PATTERN,
+            variables);
+    }
+
     private static IndexTemplateConfig anomalyDetectionResultsTemplate() {
         Map<String, String> variables = new HashMap<>();
         variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
@@ -83,10 +103,16 @@ private static IndexTemplateConfig anomalyDetectionResultsTemplate() {
             variables);
     }
 
-    private static IndexTemplateConfig statsTemplate() {
+    private static IndexTemplateConfig statsTemplate(boolean ilmEnabled) {
         Map<String, String> variables = new HashMap<>();
         variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
         variables.put("xpack.ml.stats.mappings", MlStatsIndex.mapping());
+        variables.put(
+            "xpack.ml.index.lifecycle.settings",
+            ilmEnabled
+                ? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" +
+                ",\"index.lifecycle.rollover_alias\": \"" + MlStatsIndex.writeAlias() + "\"\n"
+                : "");
 
         return new IndexTemplateConfig(MlStatsIndex.TEMPLATE_NAME,
             ROOT_RESOURCE_PATH + "stats_index_template.json",
@@ -94,9 +120,20 @@ private static IndexTemplateConfig statsTemplate() {
             variables);
     }
 
+    private final List<IndexTemplateConfig> templatesToUse;
+
     public MlIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client,
                                    NamedXContentRegistry xContentRegistry) {
         super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
+        boolean ilmEnabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings);
+        templatesToUse = Arrays.asList(
+            ANOMALY_DETECTION_RESULTS_TEMPLATE,
+            ilmEnabled ? ANOMALY_DETECTION_STATE_TEMPLATE : ANOMALY_DETECTION_STATE_TEMPLATE_NO_ILM,
+            CONFIG_TEMPLATE,
+            INFERENCE_TEMPLATE,
+            META_TEMPLATE,
+            NOTIFICATIONS_TEMPLATE,
+            ilmEnabled ? STATS_TEMPLATE : STATS_TEMPLATE_NO_ILM);
     }
 
     @Override
@@ -106,20 +143,12 @@ protected boolean requiresMasterNode() {
 
     @Override
     protected List<IndexTemplateConfig> getTemplateConfigs() {
-        return Arrays.asList(
-            ANOMALY_DETECTION_RESULTS_TEMPLATE,
-            ANOMALY_DETECTION_STATE_TEMPLATE,
-            CONFIG_TEMPLATE,
-            INFERENCE_TEMPLATE,
-            META_TEMPLATE,
-            NOTIFICATIONS_TEMPLATE,
-            STATS_TEMPLATE
-        );
+        return templatesToUse;
     }
 
     @Override
     protected List<LifecyclePolicyConfig> getPolicyConfigs() {
-        return Collections.emptyList();
+        return Collections.singletonList(ML_SIZE_BASED_ILM_POLICY);
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
new file mode 100644
index 0000000000000..d415156450082
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
+import org.elasticsearch.xpack.core.ml.job.config.Detector;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class TransportEstimateModelMemoryAction
+    extends HandledTransportAction<EstimateModelMemoryAction.Request, EstimateModelMemoryAction.Response> {
+
+    private static final Logger logger = LogManager.getLogger(TransportEstimateModelMemoryAction.class);
+
+    static final ByteSizeValue BASIC_REQUIREMENT = new ByteSizeValue(10, ByteSizeUnit.MB);
+    static final long BYTES_PER_INFLUENCER_VALUE = new ByteSizeValue(10, ByteSizeUnit.KB).getBytes();
+    private static final long BYTES_IN_MB = new ByteSizeValue(1, ByteSizeUnit.MB).getBytes();
+
+    @Inject
+    public TransportEstimateModelMemoryAction(TransportService transportService,
+                                              ActionFilters actionFilters) {
+        super(EstimateModelMemoryAction.NAME, transportService, actionFilters, EstimateModelMemoryAction.Request::new);
+    }
+
+    @Override
+    protected void doExecute(Task task,
+                             EstimateModelMemoryAction.Request request,
+                             ActionListener<EstimateModelMemoryAction.Response> listener) {
+
+        AnalysisConfig analysisConfig = request.getAnalysisConfig();
+        Map<String, Long> overallCardinality = request.getOverallCardinality();
+        Map<String, Long> maxBucketCardinality = request.getMaxBucketCardinality();
+
+        long answer = BASIC_REQUIREMENT.getBytes()
+            + calculateDetectorsRequirementBytes(analysisConfig, overallCardinality)
+            + calculateInfluencerRequirementBytes(analysisConfig, maxBucketCardinality)
+            + calculateCategorizationRequirementBytes(analysisConfig);
+
+        listener.onResponse(new EstimateModelMemoryAction.Response(roundUpToNextMb(answer)));
+    }
+
+    static long calculateDetectorsRequirementBytes(AnalysisConfig analysisConfig, Map<String, Long> overallCardinality) {
+        return analysisConfig.getDetectors().stream().map(detector -> calculateDetectorRequirementBytes(detector, overallCardinality))
+            .reduce(0L, Long::sum);
+    }
+
+    static long calculateDetectorRequirementBytes(Detector detector, Map<String, Long> overallCardinality) {
+
+        long answer = 0;
+
+        switch (detector.getFunction()) {
+            case COUNT:
+            case LOW_COUNT:
+            case HIGH_COUNT:
+            case NON_ZERO_COUNT:
+            case LOW_NON_ZERO_COUNT:
+            case HIGH_NON_ZERO_COUNT:
+                answer = 1; // TODO add realistic number
+                break;
+            case DISTINCT_COUNT:
+            case LOW_DISTINCT_COUNT:
+            case HIGH_DISTINCT_COUNT:
+                answer = 1; // TODO add realistic number
+                break;
+            case RARE:
+            case FREQ_RARE:
+                answer = 1; // TODO add realistic number
+                break;
+            case INFO_CONTENT:
+            case LOW_INFO_CONTENT:
+            case HIGH_INFO_CONTENT:
+                answer = 1; // TODO add realistic number
+                break;
+            case METRIC:
+                answer = 1; // TODO add realistic number
+                break;
+            case MEAN:
+            case LOW_MEAN:
+            case HIGH_MEAN:
+            case AVG:
+            case LOW_AVG:
+            case HIGH_AVG:
+            case MIN:
+            case MAX:
+            case SUM:
+            case LOW_SUM:
+            case HIGH_SUM:
+            case NON_NULL_SUM:
+            case LOW_NON_NULL_SUM:
+            case HIGH_NON_NULL_SUM:
+                // 64 comes from https://github.com/elastic/kibana/issues/18722
+                answer = new ByteSizeValue(64, ByteSizeUnit.KB).getBytes();
+                break;
+            case MEDIAN:
+            case LOW_MEDIAN:
+            case HIGH_MEDIAN:
+                answer = 1; // TODO add realistic number
+                break;
+            case VARP:
+            case LOW_VARP:
+            case HIGH_VARP:
+                answer = 1; // TODO add realistic number
+                break;
+            case TIME_OF_DAY:
+            case TIME_OF_WEEK:
+                answer = 1; // TODO add realistic number
+                break;
+            case LAT_LONG:
+                answer = 1; // TODO add realistic number
+                break;
+            default:
+                assert false : "unhandled detector function: " + detector.getFunction().getFullName();
+        }
+
+        String byFieldName = detector.getByFieldName();
+        if (byFieldName != null) {
+            answer *= cardinalityEstimate(Detector.BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName, overallCardinality, true);
+        }
+
+        String overFieldName = detector.getOverFieldName();
+        if (overFieldName != null) {
+            cardinalityEstimate(Detector.OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName, overallCardinality, true);
+            // TODO - how should "over" field cardinality affect estimate?
+        }
+
+        String partitionFieldName = detector.getPartitionFieldName();
+        if (partitionFieldName != null) {
+            answer *=
+                cardinalityEstimate(Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName, overallCardinality, true);
+        }
+
+        return answer;
+    }
+
+    static long calculateInfluencerRequirementBytes(AnalysisConfig analysisConfig, Map<String, Long> maxBucketCardinality) {
+
+        // Influencers that are also by/over/partition fields do not consume extra memory by being influencers
+        Set<String> pureInfluencers = new HashSet<>(analysisConfig.getInfluencers());
+        for (Detector detector : analysisConfig.getDetectors()) {
+            pureInfluencers.removeAll(detector.extractAnalysisFields());
+        }
+
+        return pureInfluencers.stream()
+            .map(influencer -> cardinalityEstimate(AnalysisConfig.INFLUENCERS.getPreferredName(), influencer, maxBucketCardinality, false)
+                * BYTES_PER_INFLUENCER_VALUE)
+            .reduce(0L, Long::sum);
+    }
+
+    static long calculateCategorizationRequirementBytes(AnalysisConfig analysisConfig) {
+
+        if (analysisConfig.getCategorizationFieldName() != null) {
+            return 1; // TODO add realistic number
+        } else {
+            return 0;
+        }
+    }
+
+    static long cardinalityEstimate(String description, String fieldName, Map<String, Long> suppliedCardinailityEstimates,
+                                    boolean isOverall) {
+        Long suppliedEstimate = suppliedCardinailityEstimates.get(fieldName);
+        if (suppliedEstimate != null) {
+            return suppliedEstimate;
+        }
+        // Don't expect the user to supply cardinality estimates for the mlcategory field that we create ourselves
+        if (AnalysisConfig.ML_CATEGORY_FIELD.equals(fieldName)) {
+            return isOverall ? 500 : 50;
+        }
+        logger.warn("[{}] cardinality estimate required for [{}] [{}] but not supplied",
+            isOverall ? "Overall" : "Bucket max", description, fieldName);
+        return 0;
+    }
+
+    static ByteSizeValue roundUpToNextMb(long bytes) {
+        assert bytes >= 0;
+        return new ByteSizeValue((BYTES_IN_MB - 1 + bytes) / BYTES_IN_MB, ByteSizeUnit.MB);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
index 5c48be663f117..c6d92413dd392 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
@@ -18,26 +19,41 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.Evaluation;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters;
 import org.elasticsearch.xpack.ml.utils.TypedChainTaskExecutor;
 
 import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.search.aggregations.MultiBucketConsumerService.MAX_BUCKET_SETTING;
 
 public class TransportEvaluateDataFrameAction extends HandledTransportAction<EvaluateDataFrameAction.Request,
     EvaluateDataFrameAction.Response> {
 
     private final ThreadPool threadPool;
     private final Client client;
+    private final AtomicReference<Integer> maxBuckets = new AtomicReference<>();
 
     @Inject
-    public TransportEvaluateDataFrameAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool,
-                                            Client client) {
+    public TransportEvaluateDataFrameAction(TransportService transportService,
+                                            ActionFilters actionFilters,
+                                            ThreadPool threadPool,
+                                            Client client,
+                                            ClusterService clusterService) {
         super(EvaluateDataFrameAction.NAME, transportService, actionFilters, EvaluateDataFrameAction.Request::new);
         this.threadPool = threadPool;
         this.client = client;
+        this.maxBuckets.set(MAX_BUCKET_SETTING.get(clusterService.getSettings()));
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_BUCKET_SETTING, this::setMaxBuckets);
+    }
+
+    private void setMaxBuckets(int maxBuckets) {
+        this.maxBuckets.set(maxBuckets);
     }
 
     @Override
-    protected void doExecute(Task task, EvaluateDataFrameAction.Request request,
+    protected void doExecute(Task task,
+                             EvaluateDataFrameAction.Request request,
                              ActionListener<EvaluateDataFrameAction.Response> listener) {
         ActionListener<List<Void>> resultsListener = ActionListener.wrap(
             unused -> {
@@ -48,7 +64,9 @@ protected void doExecute(Task task, EvaluateDataFrameAction.Request request,
             listener::onFailure
         );
 
-        EvaluationExecutor evaluationExecutor = new EvaluationExecutor(threadPool, client, request);
+        // Create an immutable collection of parameters to be used by evaluation metrics.
+        EvaluationParameters parameters = new EvaluationParameters(maxBuckets.get());
+        EvaluationExecutor evaluationExecutor = new EvaluationExecutor(threadPool, client, parameters, request);
         evaluationExecutor.execute(resultsListener);
     }
 
@@ -68,12 +86,14 @@ protected void doExecute(Task task, EvaluateDataFrameAction.Request request,
     private static final class EvaluationExecutor extends TypedChainTaskExecutor<Void> {
 
         private final Client client;
+        private final EvaluationParameters parameters;
         private final EvaluateDataFrameAction.Request request;
         private final Evaluation evaluation;
 
-        EvaluationExecutor(ThreadPool threadPool, Client client, EvaluateDataFrameAction.Request request) {
+        EvaluationExecutor(ThreadPool threadPool, Client client, EvaluationParameters parameters, EvaluateDataFrameAction.Request request) {
             super(threadPool.generic(), unused -> true, unused -> true);
             this.client = client;
+            this.parameters = parameters;
             this.request = request;
             this.evaluation = request.getEvaluation();
             // Add one task only. Other tasks will be added as needed by the nextTask method itself.
@@ -82,7 +102,7 @@ private static final class EvaluationExecutor extends TypedChainTaskExecutor<Voi
 
         private TypedChainTaskExecutor.ChainTask<Void> nextTask() {
             return listener -> {
-                SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(request.getParsedQuery());
+                SearchSourceBuilder searchSourceBuilder = evaluation.buildSearch(parameters, request.getParsedQuery());
                 SearchRequest searchRequest = new SearchRequest(request.getIndices()).source(searchSourceBuilder);
                 client.execute(
                     SearchAction.INSTANCE,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
index 052f3ed4ddca7..ca80f6fd8e72c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
@@ -7,6 +7,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
@@ -97,6 +98,7 @@ public class TransportStartDataFrameAnalyticsAction
     extends TransportMasterNodeAction<StartDataFrameAnalyticsAction.Request, AcknowledgedResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportStartDataFrameAnalyticsAction.class);
+    private static final String PRIMARY_SHARDS_INACTIVE = "not all primary shards are active";
 
     private final XPackLicenseState licenseState;
     private final Client client;
@@ -410,8 +412,26 @@ public void onFailure(Exception e) {
 
                 @Override
                 public void onTimeout(TimeValue timeout) {
-                    listener.onFailure(new ElasticsearchException(
-                        "Starting data frame analytics [" + task.getParams().getId() + "] timed out after [" + timeout + "]"));
+                    logger.error(
+                        () -> new ParameterizedMessage("[{}] timed out when starting task after [{}]. Assignment explanation [{}]",
+                            task.getParams().getId(),
+                            timeout,
+                            predicate.assignmentExplanation));
+                    if (predicate.assignmentExplanation != null) {
+                        cancelAnalyticsStart(task,
+                            new ElasticsearchStatusException(
+                                "Could not start data frame analytics task, timed out after [{}] waiting for task assignment. "
+                                    + "Assignment explanation [{}]",
+                                RestStatus.TOO_MANY_REQUESTS,
+                                timeout,
+                                predicate.assignmentExplanation),
+                            listener);
+                    } else {
+                        listener.onFailure(new ElasticsearchException(
+                            "Starting data frame analytics [{}] timed out after [{}]",
+                            task.getParams().getId(),
+                            timeout));
+                    }
                 }
         });
     }
@@ -436,6 +456,7 @@ private StartContext(DataFrameAnalyticsConfig config, List<PhaseProgress> progre
     private static class AnalyticsPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
 
         private volatile Exception exception;
+        private volatile String assignmentExplanation;
 
         @Override
         public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTask) {
@@ -450,9 +471,15 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTa
                 return true;
             }
 
-            if (assignment != null && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false &&
-                assignment.isAssigned() == false) {
-                // Assignment has failed despite passing our "fast fail" validation
+            if (assignment != null
+                && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false
+                && assignment.isAssigned() == false) {
+                assignmentExplanation = assignment.getExplanation();
+                // Assignment failed due to primary shard check.
+                // This is hopefully intermittent and we should allow another assignment attempt.
+                if (assignmentExplanation.contains(PRIMARY_SHARDS_INACTIVE)) {
+                    return false;
+                }
                 exception = new ElasticsearchStatusException("Could not start data frame analytics task, allocation explanation [" +
                     assignment.getExplanation() + "]", RestStatus.TOO_MANY_REQUESTS);
                 return true;
@@ -583,8 +610,12 @@ public PersistentTasksCustomMetaData.Assignment getAssignment(StartDataFrameAnal
                     MlStatsIndex.indexPattern(),
                     AnomalyDetectorsIndex.jobStateIndexPattern());
             if (unavailableIndices.size() != 0) {
-                String reason = "Not opening data frame analytics job [" + id +
-                    "], because not all primary shards are active for the following indices [" + String.join(",", unavailableIndices) + "]";
+                String reason = "Not opening data frame analytics job ["
+                    + id
+                    + "], because "
+                    + PRIMARY_SHARDS_INACTIVE
+                    + " for the following indices ["
+                    + String.join(",", unavailableIndices) + "]";
                 logger.debug(reason);
                 return new PersistentTasksCustomMetaData.Assignment(null, reason);
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
index 8202c0ef3d28e..eb13f2395ef8d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
@@ -14,6 +14,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -27,7 +28,7 @@ public class TimeBasedExtractedFields extends ExtractedFields {
     private final ExtractedField timeField;
 
     public TimeBasedExtractedFields(ExtractedField timeField, List<ExtractedField> allFields) {
-        super(allFields);
+        super(allFields, Collections.emptyMap());
         if (!allFields.contains(timeField)) {
             throw new IllegalArgumentException("timeField should also be contained in allFields");
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
index 6d9b6fb04b839..ba1bdf9a6a5d0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
@@ -234,6 +234,10 @@ public List<String> getFieldNames() {
         return context.extractedFields.getAllFields().stream().map(ExtractedField::getName).collect(Collectors.toList());
     }
 
+    public List<ExtractedField> getAllExtractedFields() {
+        return context.extractedFields.getAllFields();
+    }
+
     public DataSummary collectDataSummary() {
         SearchRequestBuilder searchRequestBuilder = buildDataSummarySearchRequestBuilder();
         SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
index 0c885806d730c..2254a3de0a60c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
@@ -58,15 +58,15 @@ public class ExtractedFieldsDetector {
     private final DataFrameAnalyticsConfig config;
     private final int docValueFieldsLimit;
     private final FieldCapabilitiesResponse fieldCapabilitiesResponse;
-    private final Map<String, Long> fieldCardinalities;
+    private final Map<String, Long> cardinalitiesForFieldsWithConstraints;
 
     ExtractedFieldsDetector(String[] index, DataFrameAnalyticsConfig config, int docValueFieldsLimit,
-                            FieldCapabilitiesResponse fieldCapabilitiesResponse, Map<String, Long> fieldCardinalities) {
+                            FieldCapabilitiesResponse fieldCapabilitiesResponse, Map<String, Long> cardinalitiesForFieldsWithConstraints) {
         this.index = Objects.requireNonNull(index);
         this.config = Objects.requireNonNull(config);
         this.docValueFieldsLimit = docValueFieldsLimit;
         this.fieldCapabilitiesResponse = Objects.requireNonNull(fieldCapabilitiesResponse);
-        this.fieldCardinalities = Objects.requireNonNull(fieldCardinalities);
+        this.cardinalitiesForFieldsWithConstraints = Objects.requireNonNull(cardinalitiesForFieldsWithConstraints);
     }
 
     public Tuple<ExtractedFields, List<FieldSelection>> detect() {
@@ -286,12 +286,13 @@ private void checkRequiredFields(Set<String> fields) {
 
     private void checkFieldsWithCardinalityLimit() {
         for (FieldCardinalityConstraint constraint : config.getAnalysis().getFieldCardinalityConstraints()) {
-            constraint.check(fieldCardinalities.get(constraint.getField()));
+            constraint.check(cardinalitiesForFieldsWithConstraints.get(constraint.getField()));
         }
     }
 
     private ExtractedFields detectExtractedFields(Set<String> fields, Set<FieldSelection> fieldSelection) {
-        ExtractedFields extractedFields = ExtractedFields.build(fields, Collections.emptySet(), fieldCapabilitiesResponse);
+        ExtractedFields extractedFields = ExtractedFields.build(fields, Collections.emptySet(), fieldCapabilitiesResponse,
+            cardinalitiesForFieldsWithConstraints);
         boolean preferSource = extractedFields.getDocValueFields().size() > docValueFieldsLimit;
         extractedFields = deduplicateMultiFields(extractedFields, preferSource, fieldSelection);
         if (preferSource) {
@@ -321,7 +322,7 @@ private ExtractedFields deduplicateMultiFields(ExtractedFields extractedFields,
                     chooseMultiFieldOrParent(preferSource, requiredFields, parent, multiField, fieldSelection));
             }
         }
-        return new ExtractedFields(new ArrayList<>(nameOrParentToField.values()));
+        return new ExtractedFields(new ArrayList<>(nameOrParentToField.values()), cardinalitiesForFieldsWithConstraints);
     }
 
     private ExtractedField chooseMultiFieldOrParent(boolean preferSource, Set<String> requiredFields, ExtractedField parent,
@@ -372,7 +373,7 @@ private ExtractedFields fetchFromSourceIfSupported(ExtractedFields extractedFiel
         for (ExtractedField field : extractedFields.getAllFields()) {
             adjusted.add(field.supportsFromSource() ? field.newFromSource() : field);
         }
-        return new ExtractedFields(adjusted);
+        return new ExtractedFields(adjusted, cardinalitiesForFieldsWithConstraints);
     }
 
     private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFields) {
@@ -389,7 +390,7 @@ private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFi
                 adjusted.add(field);
             }
         }
-        return new ExtractedFields(adjusted);
+        return new ExtractedFields(adjusted, cardinalitiesForFieldsWithConstraints);
     }
 
     private void addIncludedFields(ExtractedFields extractedFields, Set<FieldSelection> fieldSelection) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
index 714f63091801f..0daec5365ff48 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
@@ -14,10 +14,9 @@
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
-import static java.util.stream.Collectors.toMap;
-
 public class AnalyticsProcessConfig implements ToXContentObject {
 
     private static final String JOB_ID = "job_id";
@@ -93,12 +92,31 @@ private DataFrameAnalysisWrapper(DataFrameAnalysis analysis, ExtractedFields ext
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("name", analysis.getWriteableName());
-            builder.field(
-                "parameters",
-                analysis.getParams(
-                    extractedFields.getAllFields().stream().collect(toMap(ExtractedField::getName, ExtractedField::getTypes))));
+            builder.field("parameters", analysis.getParams(new AnalysisFieldInfo(extractedFields)));
             builder.endObject();
             return builder;
         }
     }
+
+    private static class AnalysisFieldInfo implements DataFrameAnalysis.FieldInfo {
+
+        private final ExtractedFields extractedFields;
+
+        AnalysisFieldInfo(ExtractedFields extractedFields) {
+            this.extractedFields = Objects.requireNonNull(extractedFields);
+        }
+
+        @Override
+        public Set<String> getTypes(String field) {
+            Optional<ExtractedField> extractedField = extractedFields.getAllFields().stream()
+                .filter(f -> f.getName().equals(field))
+                .findAny();
+            return extractedField.isPresent() ? extractedField.get().getTypes() : null;
+        }
+
+        @Override
+        public Long getCardinality(String field) {
+            return extractedFields.getCardinalitiesForFieldsWithConstraints().get(field);
+        }
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
index b967e1d47118c..255ecfd77c021 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
@@ -434,7 +434,7 @@ private AnalyticsResultProcessor createResultProcessor(DataFrameAnalyticsTask ta
                 new DataFrameRowsJoiner(config.getId(), dataExtractorFactory.newExtractor(true), resultsPersisterService);
             return new AnalyticsResultProcessor(
                 config, dataFrameRowsJoiner, task.getStatsHolder(), trainedModelProvider, auditor, resultsPersisterService,
-                dataExtractor.get().getFieldNames());
+                dataExtractor.get().getAllExtractedFields());
         }
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
index dff78a1cc4daf..34be13967b5d2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
@@ -33,6 +33,8 @@
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
+import org.elasticsearch.xpack.ml.extractor.ExtractedField;
+import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
@@ -42,10 +44,12 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
 
@@ -71,7 +75,7 @@ public class AnalyticsResultProcessor {
     private final TrainedModelProvider trainedModelProvider;
     private final DataFrameAnalyticsAuditor auditor;
     private final ResultsPersisterService resultsPersisterService;
-    private final List<String> fieldNames;
+    private final List<ExtractedField> fieldNames;
     private final CountDownLatch completionLatch = new CountDownLatch(1);
     private volatile String failure;
     private volatile boolean isCancelled;
@@ -79,7 +83,7 @@ public class AnalyticsResultProcessor {
     public AnalyticsResultProcessor(DataFrameAnalyticsConfig analytics, DataFrameRowsJoiner dataFrameRowsJoiner,
                                     StatsHolder statsHolder, TrainedModelProvider trainedModelProvider,
                                     DataFrameAnalyticsAuditor auditor, ResultsPersisterService resultsPersisterService,
-                                    List<String> fieldNames) {
+                                    List<ExtractedField> fieldNames) {
         this.analytics = Objects.requireNonNull(analytics);
         this.dataFrameRowsJoiner = Objects.requireNonNull(dataFrameRowsJoiner);
         this.statsHolder = Objects.requireNonNull(statsHolder);
@@ -193,8 +197,12 @@ private TrainedModelConfig createTrainedModelConfig(TrainedModelDefinition.Build
         TrainedModelDefinition definition = inferenceModel.build();
         String dependentVariable = getDependentVariable();
         List<String> fieldNamesWithoutDependentVariable = fieldNames.stream()
+            .map(ExtractedField::getName)
             .filter(f -> f.equals(dependentVariable) == false)
             .collect(toList());
+        Map<String, String> defaultFieldMapping = fieldNames.stream()
+            .filter(ef -> ef instanceof MultiField && (ef.getName().equals(dependentVariable) == false))
+            .collect(Collectors.toMap(ExtractedField::getParentField, ExtractedField::getName));
         return TrainedModelConfig.builder()
             .setModelId(modelId)
             .setCreatedBy(XPackUser.NAME)
@@ -210,6 +218,7 @@ private TrainedModelConfig createTrainedModelConfig(TrainedModelDefinition.Build
             .setParsedDefinition(inferenceModel)
             .setInput(new TrainedModelInput(fieldNamesWithoutDependentVariable))
             .setLicenseLevel(License.OperationMode.PLATINUM.description())
+            .setDefaultFieldMap(defaultFieldMapping)
             .build();
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
index 3a36bb7ff76d0..ab314a5d21851 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
@@ -28,12 +28,14 @@ public class ExtractedFields {
     private final List<ExtractedField> allFields;
     private final List<ExtractedField> docValueFields;
     private final String[] sourceFields;
+    private final Map<String, Long> cardinalitiesForFieldsWithConstraints;
 
-    public ExtractedFields(List<ExtractedField> allFields) {
+    public ExtractedFields(List<ExtractedField> allFields, Map<String, Long> cardinalitiesForFieldsWithConstraints) {
         this.allFields = Collections.unmodifiableList(allFields);
         this.docValueFields = filterFields(ExtractedField.Method.DOC_VALUE, allFields);
         this.sourceFields = filterFields(ExtractedField.Method.SOURCE, allFields).stream().map(ExtractedField::getSearchField)
             .toArray(String[]::new);
+        this.cardinalitiesForFieldsWithConstraints = Collections.unmodifiableMap(cardinalitiesForFieldsWithConstraints);
     }
 
     public List<ExtractedField> getAllFields() {
@@ -48,14 +50,20 @@ public List<ExtractedField> getDocValueFields() {
         return docValueFields;
     }
 
+    public Map<String, Long> getCardinalitiesForFieldsWithConstraints() {
+        return cardinalitiesForFieldsWithConstraints;
+    }
+
     private static List<ExtractedField> filterFields(ExtractedField.Method method, List<ExtractedField> fields) {
         return fields.stream().filter(field -> field.getMethod() == method).collect(Collectors.toList());
     }
 
     public static ExtractedFields build(Collection<String> allFields, Set<String> scriptFields,
-                                        FieldCapabilitiesResponse fieldsCapabilities) {
+                                        FieldCapabilitiesResponse fieldsCapabilities,
+                                        Map<String, Long> cardinalitiesForFieldsWithConstraints) {
         ExtractionMethodDetector extractionMethodDetector = new ExtractionMethodDetector(scriptFields, fieldsCapabilities);
-        return new ExtractedFields(allFields.stream().map(field -> extractionMethodDetector.detect(field)).collect(Collectors.toList()));
+        return new ExtractedFields(allFields.stream().map(field -> extractionMethodDetector.detect(field)).collect(Collectors.toList()),
+            cardinalitiesForFieldsWithConstraints);
     }
 
     public static TimeField newTimeField(String name, ExtractedField.Method method) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java
index e41905a5faddd..d856022c09d69 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/MultiField.java
@@ -17,7 +17,7 @@ public class MultiField implements ExtractedField {
     private final ExtractedField field;
     private final String parent;
 
-    MultiField(String parent, ExtractedField field) {
+    public MultiField(String parent, ExtractedField field) {
         this(field.getName(), field.getSearchField(), parent, field);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index d36aa211b1e9d..2a4861c5247f7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -34,7 +35,7 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.MapHelper;
+import org.elasticsearch.xpack.ml.inference.loadingservice.Model;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
 
 import java.util.Arrays;
@@ -63,6 +64,7 @@ public class InferenceProcessor extends AbstractProcessor {
     public static final String INFERENCE_CONFIG = "inference_config";
     public static final String TARGET_FIELD = "target_field";
     public static final String FIELD_MAPPINGS = "field_mappings";
+    public static final String FIELD_MAP = "field_map";
     private static final String DEFAULT_TARGET_FIELD = "ml.inference";
 
     private final Client client;
@@ -70,7 +72,7 @@ public class InferenceProcessor extends AbstractProcessor {
 
     private final String targetField;
     private final InferenceConfig inferenceConfig;
-    private final Map<String, String> fieldMapping;
+    private final Map<String, String> fieldMap;
     private final InferenceAuditor auditor;
     private volatile boolean previouslyLicensed;
     private final AtomicBoolean shouldAudit = new AtomicBoolean(true);
@@ -81,14 +83,14 @@ public InferenceProcessor(Client client,
                               String targetField,
                               String modelId,
                               InferenceConfig inferenceConfig,
-                              Map<String, String> fieldMapping) {
+                              Map<String, String> fieldMap) {
         super(tag);
         this.client = ExceptionsHelper.requireNonNull(client, "client");
         this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD);
         this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor");
         this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
         this.inferenceConfig = ExceptionsHelper.requireNonNull(inferenceConfig, INFERENCE_CONFIG);
-        this.fieldMapping = ExceptionsHelper.requireNonNull(fieldMapping, FIELD_MAPPINGS);
+        this.fieldMap = ExceptionsHelper.requireNonNull(fieldMap, FIELD_MAP);
     }
 
     public String getModelId() {
@@ -126,14 +128,7 @@ void handleResponse(InternalInferModelAction.Response response,
 
     InternalInferModelAction.Request buildRequest(IngestDocument ingestDocument) {
         Map<String, Object> fields = new HashMap<>(ingestDocument.getSourceAndMetadata());
-        if (fieldMapping != null) {
-            fieldMapping.forEach((src, dest) -> {
-                Object srcValue = MapHelper.dig(src, fields);
-                if (srcValue != null) {
-                    fields.put(dest, srcValue);
-                }
-            });
-        }
+        Model.mapFieldsIfNecessary(fields, fieldMap);
         return new InternalInferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed);
     }
 
@@ -242,7 +237,14 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
             // If multiple inference processors are in the same pipeline, it is wise to tag them
             // The tag will keep default value entries from stepping on each other
             String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, TARGET_FIELD, defaultTargetField);
-            Map<String, String> fieldMapping = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
+            Map<String, String> fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAP);
+            if (fieldMap == null) {
+                fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
+                //TODO Remove in 8.x
+                if (fieldMap != null) {
+                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
+                }
+            }
             InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
 
             return new InferenceProcessor(client,
@@ -251,7 +253,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
                 targetField,
                 modelId,
                 inferenceConfig,
-                fieldMapping);
+                fieldMap);
         }
 
         // Package private for testing
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java
index 8a233d534a76f..f77188853ad96 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
 import org.elasticsearch.xpack.core.ml.utils.MapHelper;
 
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -28,11 +29,16 @@ public class LocalModel implements Model {
     private final TrainedModelDefinition trainedModelDefinition;
     private final String modelId;
     private final Set<String> fieldNames;
+    private final Map<String, String> defaultFieldMap;
 
-    public LocalModel(String modelId, TrainedModelDefinition trainedModelDefinition, TrainedModelInput input) {
+    public LocalModel(String modelId,
+                      TrainedModelDefinition trainedModelDefinition,
+                      TrainedModelInput input,
+                      Map<String, String> defaultFieldMap) {
         this.trainedModelDefinition = trainedModelDefinition;
         this.modelId = modelId;
         this.fieldNames = new HashSet<>(input.getFieldNames());
+        this.defaultFieldMap = defaultFieldMap == null ? null : new HashMap<>(defaultFieldMap);
     }
 
     long ramBytesUsed() {
@@ -61,6 +67,7 @@ public String getResultsType() {
     @Override
     public void infer(Map<String, Object> fields, InferenceConfig config, ActionListener<InferenceResults> listener) {
         try {
+            Model.mapFieldsIfNecessary(fields, defaultFieldMap);
             if (fieldNames.stream().allMatch(f -> MapHelper.dig(f, fields) == null)) {
                 listener.onResponse(new WarningInferenceResults(Messages.getMessage(INFERENCE_WARNING_ALL_FIELDS_MISSING, modelId)));
                 return;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java
index fb32ce7f646df..f1df6908d6329 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/Model.java
@@ -8,6 +8,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig;
+import org.elasticsearch.xpack.core.ml.utils.MapHelper;
 
 import java.util.Map;
 
@@ -18,4 +19,27 @@ public interface Model {
     void infer(Map<String, Object> fields, InferenceConfig inferenceConfig, ActionListener<InferenceResults> listener);
 
     String getModelId();
+
+    /**
+     * Used for translating field names in according to the passed `fieldMappings` parameter.
+     *
+     * This mutates the `fields` parameter in-place.
+     *
+     * Fields are only appended. If the expected field name already exists, it is not created/overwritten.
+     *
+     * Original fields are not deleted.
+     *
+     * @param fields Fields to map against
+     * @param fieldMapping Field originalName to expectedName string mapping
+     */
+    static void mapFieldsIfNecessary(Map<String, Object> fields, Map<String, String> fieldMapping) {
+        if (fieldMapping != null) {
+            fieldMapping.forEach((src, dest) -> {
+                Object srcValue = MapHelper.dig(src, fields);
+                if (srcValue != null) {
+                    fields.putIfAbsent(dest, srcValue);
+                }
+            });
+        }
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
index b5862acfefc27..0082549f088e3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
@@ -142,7 +142,8 @@ public void getModel(String modelId, ActionListener<Model> modelActionListener)
                     modelActionListener.onResponse(new LocalModel(
                         trainedModelConfig.getModelId(),
                         trainedModelConfig.ensureParsedDefinition(namedXContentRegistry).getModelDefinition(),
-                        trainedModelConfig.getInput())),
+                        trainedModelConfig.getInput(),
+                        trainedModelConfig.getDefaultFieldMap())),
                 modelActionListener::onFailure
             ));
         } else {
@@ -200,7 +201,8 @@ private void handleLoadSuccess(String modelId, TrainedModelConfig trainedModelCo
         LocalModel loadedModel = new LocalModel(
             trainedModelConfig.getModelId(),
             trainedModelConfig.ensureParsedDefinition(namedXContentRegistry).getModelDefinition(),
-            trainedModelConfig.getInput());
+            trainedModelConfig.getInput(),
+            trainedModelConfig.getDefaultFieldMap());
         synchronized (loadingListeners) {
             listeners = loadingListeners.remove(modelId);
             // If there is no loadingListener that means the loading was canceled and the listener was already notified as such
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java
new file mode 100644
index 0000000000000..319446e9fd169
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestEstimateModelMemoryAction.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.job;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestEstimateModelMemoryAction extends BaseRestHandler {
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(new Route(POST, MachineLearning.BASE_PATH + "anomaly_detectors/_estimate_model_memory"));
+    }
+
+    @Override
+    public String getName() {
+        return "ml_estimate_model_memory_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        EstimateModelMemoryAction.Request request =
+            EstimateModelMemoryAction.Request.parseRequest(restRequest.contentOrSourceParamParser());
+        return channel -> client.execute(EstimateModelMemoryAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
index 0925d252e2019..dfbc4be8e8265 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
@@ -484,7 +484,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() throws Excep
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"modelprocessorlicensetest\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }]}\n";
         // Creating a pipeline should work
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java
new file mode 100644
index 0000000000000..d339c54815a76
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java
@@ -0,0 +1,181 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.ilm.LifecycleAction;
+import org.elasticsearch.xpack.core.ilm.RolloverAction;
+import org.elasticsearch.xpack.core.ml.MlStatsIndex;
+import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+import org.mockito.stubbing.Answer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.mock.orig.Mockito.verify;
+import static org.elasticsearch.mock.orig.Mockito.when;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+
+public class MlIndexTemplateRegistryTests extends ESTestCase {
+
+    private final DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
+    private final DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
+
+    private NamedXContentRegistry xContentRegistry;
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+    private Client client;
+    private ArgumentCaptor<PutIndexTemplateRequest> putIndexTemplateRequestCaptor;
+
+    @Before
+    public void setUpMocks() {
+        threadPool = mock(ThreadPool.class);
+        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+        when(threadPool.generic()).thenReturn(EsExecutors.newDirectExecutorService());
+
+        client = mock(Client.class);
+        when(client.threadPool()).thenReturn(threadPool);
+        AdminClient adminClient = mock(AdminClient.class);
+        IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class);
+        when(adminClient.indices()).thenReturn(indicesAdminClient);
+        when(client.admin()).thenReturn(adminClient);
+        doAnswer(withResponse(new AcknowledgedResponse(true))).when(indicesAdminClient).putTemplate(any(), any());
+
+        clusterService = mock(ClusterService.class);
+
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse));
+        xContentRegistry = new NamedXContentRegistry(entries);
+
+        putIndexTemplateRequestCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
+    }
+
+    public void testStateTemplateWithIlm() {
+        MlIndexTemplateRegistry registry =
+            new MlIndexTemplateRegistry(
+                Settings.builder()
+                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true)
+                    .build(),
+                clusterService, threadPool, client, xContentRegistry);
+
+        registry.clusterChanged(createClusterChangedEvent(nodes));
+
+        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
+
+        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
+            .filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX))
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("expected the ml state index template to be put"));
+        assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy"));
+        assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-state-write"));
+    }
+
+    public void testStateTemplateWithNoIlm() {
+        MlIndexTemplateRegistry registry =
+            new MlIndexTemplateRegistry(
+                Settings.builder()
+                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false)
+                    .build(),
+                clusterService, threadPool, client, xContentRegistry);
+
+        registry.clusterChanged(createClusterChangedEvent(nodes));
+
+        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
+
+        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
+            .filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX))
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("expected the ml state index template to be put"));
+        assertThat(req.settings().get("index.lifecycle.name"), is(nullValue()));
+        assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue()));
+    }
+
+    public void testStatsTemplateWithIlm() {
+        MlIndexTemplateRegistry registry =
+            new MlIndexTemplateRegistry(
+                Settings.builder()
+                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true)
+                    .build(),
+                clusterService, threadPool, client, xContentRegistry);
+
+        registry.clusterChanged(createClusterChangedEvent(nodes));
+
+        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
+
+        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
+            .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME))
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("expected the ml stats index template to be put"));
+        assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy"));
+        assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-stats-write"));
+    }
+
+    public void testStatsTemplateWithNoIlm() {
+        MlIndexTemplateRegistry registry =
+            new MlIndexTemplateRegistry(
+                Settings.builder()
+                    .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false)
+                    .build(),
+                clusterService, threadPool, client, xContentRegistry);
+
+        registry.clusterChanged(createClusterChangedEvent(nodes));
+
+        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
+
+        PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
+            .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME))
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("expected the ml stats index template to be put"));
+        assertThat(req.settings().get("index.lifecycle.name"), is(nullValue()));
+        assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue()));
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <Response> Answer<Response> withResponse(Response response) {
+        return invocationOnMock -> {
+            ActionListener<Response> listener = (ActionListener<Response>) invocationOnMock.getArguments()[1];
+            listener.onResponse(response);
+            return null;
+        };
+    }
+
+    private static ClusterChangedEvent createClusterChangedEvent(DiscoveryNodes nodes) {
+        return new ClusterChangedEvent(
+            "created-from-test",
+            ClusterState.builder(new ClusterName("test")).nodes(nodes).build(),
+            ClusterState.builder(new ClusterName("test")).build());
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
index 734533cac54b0..3d08d8626b5ce 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
@@ -15,7 +15,9 @@
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -44,6 +46,8 @@ protected Settings nodeSettings() {
         newSettings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
         newSettings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         newSettings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
+        // Disable ILM history index so that the tests don't have to clean it up
+        newSettings.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
         return newSettings.build();
     }
 
@@ -55,7 +59,10 @@ protected NamedXContentRegistry xContentRegistry() {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(LocalStateMachineLearning.class);
+        return pluginList(
+            LocalStateMachineLearning.class,
+            // ILM is required for .ml-state template index settings
+            IndexLifecycle.class);
     }
 
     /**
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
new file mode 100644
index 0000000000000..ea10c9fb5f64c
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
+import org.elasticsearch.xpack.core.ml.job.config.Detector;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class TransportEstimateModelMemoryActionTests extends ESTestCase {
+
+    public void testCalculateDetectorRequirementBytes() {
+
+        Map<String, Long> overallCardinality = new HashMap<>();
+        overallCardinality.put("part", 100L);
+        overallCardinality.put("buy", 200L);
+        overallCardinality.put("ovr", 300L);
+
+        String function = randomFrom("mean", "min", "max", "sum");
+
+        Detector noSplit = createDetector(function, "field", null, null, null);
+        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(noSplit,
+            overallCardinality), is(65536L));
+
+        Detector withByField = createDetector(function, "field", "buy", null, null);
+        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withByField,
+            overallCardinality), is(200 * 65536L));
+
+        Detector withPartitionField = createDetector(function, "field", null, null, "part");
+        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withPartitionField,
+            overallCardinality), is(100 * 65536L));
+
+        Detector withByAndPartitionFields = createDetector(function, "field", "buy", null, "part");
+        assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withByAndPartitionFields,
+            overallCardinality), is(200 * 100 * 65536L));
+    }
+
+    public void testCalculateInfluencerRequirementBytes() {
+
+        Map<String, Long> maxBucketCardinality = new HashMap<>();
+        maxBucketCardinality.put("part", 100L);
+        maxBucketCardinality.put("inf1", 200L);
+        maxBucketCardinality.put("inf2", 300L);
+
+        AnalysisConfig noInfluencers = createCountAnalysisConfig(null, null);
+        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(noInfluencers,
+            maxBucketCardinality), is(0L));
+
+        AnalysisConfig influencerAlsoPartitionField = createCountAnalysisConfig(null, "part", "part");
+        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(influencerAlsoPartitionField,
+            maxBucketCardinality), is(0L));
+
+        AnalysisConfig influencerNotPartitionField = createCountAnalysisConfig(null, "part", "inf1");
+        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(influencerNotPartitionField,
+            maxBucketCardinality), is(200 * TransportEstimateModelMemoryAction.BYTES_PER_INFLUENCER_VALUE));
+
+        AnalysisConfig otherInfluencerAsWellAsPartitionField = createCountAnalysisConfig(null, "part", "part", "inf1");
+        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(otherInfluencerAsWellAsPartitionField,
+            maxBucketCardinality), is(200 * TransportEstimateModelMemoryAction.BYTES_PER_INFLUENCER_VALUE));
+
+        AnalysisConfig twoInfluencersNotPartitionField = createCountAnalysisConfig(null, "part", "part", "inf1", "inf2");
+        assertThat(TransportEstimateModelMemoryAction.calculateInfluencerRequirementBytes(twoInfluencersNotPartitionField,
+            maxBucketCardinality), is((200 + 300) * TransportEstimateModelMemoryAction.BYTES_PER_INFLUENCER_VALUE));
+    }
+
+    public void testCalculateCategorizationRequirementBytes() {
+
+        AnalysisConfig analysisConfigWithoutCategorization = createCountAnalysisConfig(null, null);
+        assertThat(TransportEstimateModelMemoryAction.calculateCategorizationRequirementBytes(analysisConfigWithoutCategorization), is(0L));
+
+        AnalysisConfig analysisConfigWithCategorization = createCountAnalysisConfig(randomAlphaOfLength(10), null);
+        assertThat(TransportEstimateModelMemoryAction.calculateCategorizationRequirementBytes(analysisConfigWithCategorization), is(1L));
+    }
+
+    public void testRoundUpToNextMb() {
+
+        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(0),
+            equalTo(new ByteSizeValue(0, ByteSizeUnit.BYTES)));
+        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(1),
+            equalTo(new ByteSizeValue(1, ByteSizeUnit.MB)));
+        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(randomIntBetween(1, 1024 * 1024)),
+            equalTo(new ByteSizeValue(1, ByteSizeUnit.MB)));
+        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(1024 * 1024),
+            equalTo(new ByteSizeValue(1, ByteSizeUnit.MB)));
+        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(1024 * 1024 + 1),
+            equalTo(new ByteSizeValue(2, ByteSizeUnit.MB)));
+        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(2 * 1024 * 1024),
+            equalTo(new ByteSizeValue(2, ByteSizeUnit.MB)));
+    }
+
+    public static Detector createDetector(String function, String fieldName, String byFieldName,
+                                          String overFieldName, String partitionFieldName) {
+
+        Detector.Builder detectorBuilder = new Detector.Builder(function, fieldName);
+        detectorBuilder.setByFieldName(byFieldName);
+        detectorBuilder.setOverFieldName(overFieldName);
+        detectorBuilder.setPartitionFieldName(partitionFieldName);
+        return detectorBuilder.build();
+    }
+
+    public static AnalysisConfig createCountAnalysisConfig(String categorizationFieldName, String partitionFieldName,
+                                                           String... influencerFieldNames) {
+
+        Detector.Builder detectorBuilder = new Detector.Builder("count", null);
+        detectorBuilder.setPartitionFieldName((categorizationFieldName != null) ? AnalysisConfig.ML_CATEGORY_FIELD : partitionFieldName);
+
+        AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(detectorBuilder.build()));
+
+        if (categorizationFieldName != null) {
+            builder.setCategorizationFieldName(categorizationFieldName);
+        }
+
+        if (influencerFieldNames.length > 0) {
+            builder.setInfluencers(Arrays.asList(influencerFieldNames));
+        }
+
+        return builder.build();
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
index 03a011c717292..d776ffd9818d9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
@@ -267,7 +267,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
                     new HashMap<String, Object>() {{
                         put(InferenceProcessor.MODEL_ID, modelId);
                         put("inference_config", Collections.singletonMap("regression", Collections.emptyMap()));
-                        put("field_mappings", Collections.emptyMap());
+                        put("field_map", Collections.emptyMap());
                         put("target_field", randomAlphaOfLength(10));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId + num,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
index d54997f1dbdc3..b75392e03c2bb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
@@ -81,7 +81,7 @@ public void setUpTests() {
         query = QueryBuilders.matchAllQuery();
         extractedFields = new ExtractedFields(Arrays.asList(
             new DocValueField("field_1", Collections.singleton("keyword")),
-            new DocValueField("field_2", Collections.singleton("keyword"))));
+            new DocValueField("field_2", Collections.singleton("keyword"))), Collections.emptyMap());
         scrollSize = 1000;
         headers = Collections.emptyMap();
 
@@ -299,7 +299,7 @@ public void testIncludeSourceIsFalseAndAtLeastOneSourceField() throws IOExceptio
         // Explicit cast of ExtractedField args necessary for Eclipse due to https://bugs.eclipse.org/bugs/show_bug.cgi?id=530915
         extractedFields = new ExtractedFields(Arrays.asList(
             (ExtractedField) new DocValueField("field_1", Collections.singleton("keyword")),
-            (ExtractedField) new SourceField("field_2", Collections.singleton("text"))));
+            (ExtractedField) new SourceField("field_2", Collections.singleton("text"))), Collections.emptyMap());
 
         TestExtractor dataExtractor = createExtractor(false, false);
 
@@ -404,7 +404,7 @@ public void testGetCategoricalFields() {
             (ExtractedField) new DocValueField("field_integer", Collections.singleton("integer")),
             (ExtractedField) new DocValueField("field_long", Collections.singleton("long")),
             (ExtractedField) new DocValueField("field_keyword", Collections.singleton("keyword")),
-            (ExtractedField) new SourceField("field_text", Collections.singleton("text"))));
+            (ExtractedField) new SourceField("field_text", Collections.singleton("text"))), Collections.emptyMap());
         TestExtractor dataExtractor = createExtractor(true, true);
 
         assertThat(dataExtractor.getCategoricalFields(OutlierDetectionTests.createRandom()), empty());
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
index 49a302a498b82..a7a5784c452f5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
@@ -294,10 +294,10 @@ public void testDetect_GivenClassificationAndDependentVariableHasInvalidCardinal
             .build();
 
         ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(SOURCE_INDEX,
-            buildClassificationConfig("some_keyword"), 100, fieldCapabilities, Collections.singletonMap("some_keyword", 3L));
+            buildClassificationConfig("some_keyword"), 100, fieldCapabilities, Collections.singletonMap("some_keyword", 31L));
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect);
 
-        assertThat(e.getMessage(), equalTo("Field [some_keyword] must have at most [2] distinct values but there were at least [3]"));
+        assertThat(e.getMessage(), equalTo("Field [some_keyword] must have at most [30] distinct values but there were at least [31]"));
     }
 
     public void testDetect_GivenIgnoredField() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
new file mode 100644
index 0000000000000..a4db8de032af5
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
+import org.elasticsearch.xpack.ml.extractor.DocValueField;
+import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+
+public class AnalyticsProcessConfigTests extends ESTestCase {
+
+    private String jobId;
+    private long rows;
+    private int cols;
+    private ByteSizeValue memoryLimit;
+    private int threads;
+    private String resultsField;
+    private Set<String> categoricalFields;
+
+    @Before
+    public void setUpConfigParams() {
+        jobId = randomAlphaOfLength(10);
+        rows = randomNonNegativeLong();
+        cols = randomIntBetween(1, 42000);
+        memoryLimit = new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES);
+        threads = randomIntBetween(1, 8);
+        resultsField = randomAlphaOfLength(10);
+
+        int categoricalFieldsSize = randomIntBetween(0, 5);
+        categoricalFields = new HashSet<>();
+        for (int i = 0; i < categoricalFieldsSize; i++) {
+            categoricalFields.add(randomAlphaOfLength(10));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenOutlierDetection() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float"))), Collections.emptyMap());
+        DataFrameAnalysis analysis = new OutlierDetection.Builder().build();
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "outlier_detection"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenRegression() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float")),
+            new DocValueField("test_dep_var", Collections.singleton("keyword"))), Collections.emptyMap());
+        DataFrameAnalysis analysis = new Regression("test_dep_var");
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "regression"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
+        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenClassificationAndDepVarIsKeyword() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float")),
+            new DocValueField("test_dep_var", Collections.singleton("keyword"))), Collections.singletonMap("test_dep_var", 5L));
+        DataFrameAnalysis analysis = new Classification("test_dep_var");
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "classification"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
+        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
+        assertThat(paramsAsMap, hasEntry("prediction_field_type", "string"));
+        assertThat(paramsAsMap, hasEntry("num_classes", 5));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenClassificationAndDepVarIsInteger() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float")),
+            new DocValueField("test_dep_var", Collections.singleton("integer"))), Collections.singletonMap("test_dep_var", 8L));
+        DataFrameAnalysis analysis = new Classification("test_dep_var");
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "classification"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
+        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
+        assertThat(paramsAsMap, hasEntry("prediction_field_type", "int"));
+        assertThat(paramsAsMap, hasEntry("num_classes", 8));
+    }
+
+    private AnalyticsProcessConfig createProcessConfig(DataFrameAnalysis analysis, ExtractedFields extractedFields) {
+        return new AnalyticsProcessConfig(jobId, rows, cols, memoryLimit, threads, resultsField, categoricalFields, analysis,
+            extractedFields);
+    }
+
+    private static Map<String, Object> toMap(AnalyticsProcessConfig config) throws IOException {
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            config.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            return XContentHelper.convertToMap(JsonXContent.jsonXContent, Strings.toString(builder), false);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertRandomizedFields(Map<String, Object> configAsMap) {
+        assertThat(configAsMap, hasEntry("job_id", jobId));
+        assertThat(configAsMap, hasEntry("rows", rows));
+        assertThat(configAsMap, hasEntry("cols", cols));
+        assertThat(configAsMap, hasEntry("memory_limit", memoryLimit.getBytes()));
+        assertThat(configAsMap, hasEntry("threads", threads));
+        assertThat(configAsMap, hasEntry("results_field", resultsField));
+        assertThat(configAsMap, hasKey("categorical_fields"));
+        assertThat((List<String>) configAsMap.get("categorical_fields"), containsInAnyOrder(categoricalFields.toArray()));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
index 6d29f1a066363..29105eca8174e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
@@ -169,7 +169,7 @@ public void testRunJob_Ok() {
         inOrder.verify(dataExtractor).getCategoricalFields(dataFrameAnalyticsConfig.getAnalysis());
         inOrder.verify(process).isProcessAlive();
         inOrder.verify(task).getStatsHolder();
-        inOrder.verify(dataExtractor).getFieldNames();
+        inOrder.verify(dataExtractor).getAllExtractedFields();
         inOrder.verify(executorServiceForProcess, times(2)).execute(any());  // 'processData' and 'processResults' threads
         verifyNoMoreInteractions(dataExtractor, executorServiceForProcess, process, task);
     }
@@ -227,7 +227,7 @@ public void testProcessContext_StartAndStop() throws Exception {
         inOrder.verify(dataExtractor).getCategoricalFields(dataFrameAnalyticsConfig.getAnalysis());
         inOrder.verify(process).isProcessAlive();
         inOrder.verify(task).getStatsHolder();
-        inOrder.verify(dataExtractor).getFieldNames();
+        inOrder.verify(dataExtractor).getAllExtractedFields();
         // stop
         inOrder.verify(dataExtractor).cancel();
         inOrder.verify(process).kill();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
index 044a06efbb2ce..2305116343961 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
@@ -24,7 +24,10 @@
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
+import org.elasticsearch.xpack.ml.extractor.DocValueField;
+import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
+import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
@@ -33,6 +36,7 @@
 import org.mockito.InOrder;
 import org.mockito.Mockito;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -158,10 +162,13 @@ public void testProcess_GivenInferenceModelIsStoredSuccessfully() {
             return null;
         }).when(trainedModelProvider).storeTrainedModel(any(TrainedModelConfig.class), any(ActionListener.class));
 
-        List<String> expectedFieldNames = Arrays.asList("foo", "bar", "baz");
+        List<ExtractedField> extractedFieldList = new ArrayList<>(3);
+        extractedFieldList.add(new DocValueField("foo", Collections.emptySet()));
+        extractedFieldList.add(new MultiField("bar", new DocValueField("bar.keyword", Collections.emptySet())));
+        extractedFieldList.add(new DocValueField("baz", Collections.emptySet()));
         TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder();
         givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null)));
-        AnalyticsResultProcessor resultProcessor = createResultProcessor(expectedFieldNames);
+        AnalyticsResultProcessor resultProcessor = createResultProcessor(extractedFieldList);
 
         resultProcessor.process(process);
         resultProcessor.awaitForCompletion();
@@ -177,7 +184,8 @@ public void testProcess_GivenInferenceModelIsStoredSuccessfully() {
         assertThat(storedModel.getTags(), contains(JOB_ID));
         assertThat(storedModel.getDescription(), equalTo(JOB_DESCRIPTION));
         assertThat(storedModel.getModelDefinition(), equalTo(inferenceModel.build()));
-        assertThat(storedModel.getInput().getFieldNames(), equalTo(Arrays.asList("bar", "baz")));
+        assertThat(storedModel.getDefaultFieldMap(), equalTo(Collections.singletonMap("bar", "bar.keyword")));
+        assertThat(storedModel.getInput().getFieldNames(), equalTo(Arrays.asList("bar.keyword", "baz")));
         assertThat(storedModel.getEstimatedHeapMemory(), equalTo(inferenceModel.build().ramBytesUsed()));
         assertThat(storedModel.getEstimatedOperations(), equalTo(inferenceModel.build().getTrainedModel().estimatedNumOperations()));
         Map<String, Object> metadata = storedModel.getMetadata();
@@ -235,8 +243,13 @@ private AnalyticsResultProcessor createResultProcessor() {
         return createResultProcessor(Collections.emptyList());
     }
 
-    private AnalyticsResultProcessor createResultProcessor(List<String> fieldNames) {
-        return new AnalyticsResultProcessor(
-            analyticsConfig, dataFrameRowsJoiner, statsHolder, trainedModelProvider, auditor, resultsPersisterService, fieldNames);
+    private AnalyticsResultProcessor createResultProcessor(List<ExtractedField> fieldNames) {
+        return new AnalyticsResultProcessor(analyticsConfig,
+            dataFrameRowsJoiner,
+            statsHolder,
+            trainedModelProvider,
+            auditor,
+            resultsPersisterService,
+            fieldNames);
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
index 5ac983e7d505b..a51eafd1d8b3d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
@@ -32,7 +32,7 @@ public void testAllTypesOfFields() {
         ExtractedField sourceField1 = new SourceField("src1", Collections.singleton("text"));
         ExtractedField sourceField2 = new SourceField("src2", Collections.singleton("text"));
         ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
-                docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2));
+                docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2), Collections.emptyMap());
 
         assertThat(extractedFields.getAllFields().size(), equalTo(6));
         assertThat(extractedFields.getDocValueFields().stream().map(ExtractedField::getName).toArray(String[]::new),
@@ -54,7 +54,7 @@ public void testBuildGivenMixtureOfTypes() {
         when(fieldCapabilitiesResponse.getField("airline")).thenReturn(airlineCaps);
 
         ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("time", "value", "airline", "airport"),
-            new HashSet<>(Collections.singletonList("airport")), fieldCapabilitiesResponse);
+            new HashSet<>(Collections.singletonList("airport")), fieldCapabilitiesResponse, Collections.emptyMap());
 
         assertThat(extractedFields.getDocValueFields().size(), equalTo(2));
         assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("time"));
@@ -77,7 +77,7 @@ public void testBuildGivenMultiFields() {
         when(fieldCapabilitiesResponse.getField("airport.keyword")).thenReturn(keyword);
 
         ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("airline.text", "airport.keyword"),
-                Collections.emptySet(), fieldCapabilitiesResponse);
+                Collections.emptySet(), fieldCapabilitiesResponse, Collections.emptyMap());
 
         assertThat(extractedFields.getDocValueFields().size(), equalTo(1));
         assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("airport.keyword"));
@@ -119,7 +119,7 @@ public void testBuildGivenFieldWithoutMappings() {
         FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ExtractedFields.build(
-                Collections.singletonList("value"), Collections.emptySet(), fieldCapabilitiesResponse));
+                Collections.singletonList("value"), Collections.emptySet(), fieldCapabilitiesResponse, Collections.emptyMap()));
         assertThat(e.getMessage(), equalTo("cannot retrieve field [value] because it has no mappings"));
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
index 9ed05e56c2c30..cb0519d845b19 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
@@ -131,7 +131,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             ingestService);
 
         Map<String, Object> config = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("unknown_type", Collections.emptyMap()));
@@ -143,7 +143,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]"));
 
         Map<String, Object> config2 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom"));
@@ -154,7 +154,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             equalTo("inference_config must be an object with one inference type mapped to an object."));
 
         Map<String, Object> config3 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap());
@@ -173,7 +173,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
         processorFactory.accept(builderClusterStateWithModelReferences(Version.V_7_5_0, "model1"));
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -191,7 +191,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -216,7 +216,7 @@ public void testCreateProcessor() {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -230,7 +230,7 @@ public void testCreateProcessor() {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -251,7 +251,7 @@ public void testCreateProcessorWithDuplicateFields() {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "ml");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(),
@@ -305,7 +305,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
                         put(InferenceProcessor.INFERENCE_CONFIG,
                                 Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap()));
                         put(InferenceProcessor.TARGET_FIELD, "new_field");
-                        put(InferenceProcessor.FIELD_MAPPINGS, Collections.singletonMap("source", "dest"));
+                        put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId, BytesReference.bytes(xContentBuilder), XContentType.JSON);
         }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java
index e7c0eecc01744..ccbebf6f6f738 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -41,13 +42,16 @@ public class LocalModelTests extends ESTestCase {
 
     public void testClassificationInfer() throws Exception {
         String modelId = "classification_model";
-        List<String> inputFields = Arrays.asList("field.foo", "field.bar", "categorical");
+        List<String> inputFields = Arrays.asList("field.foo.keyword", "field.bar", "categorical");
         TrainedModelDefinition definition = new TrainedModelDefinition.Builder()
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildClassification(false))
             .build();
 
-        Model model = new LocalModel(modelId, definition, new TrainedModelInput(inputFields));
+        Model model = new LocalModel(modelId,
+            definition,
+            new TrainedModelInput(inputFields),
+            Collections.singletonMap("field.foo", "field.foo.keyword"));
         Map<String, Object> fields = new HashMap<>() {{
             put("field.foo", 1.0);
             put("field.bar", 0.5);
@@ -68,7 +72,10 @@ public void testClassificationInfer() throws Exception {
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildClassification(true))
             .build();
-        model = new LocalModel(modelId, definition, new TrainedModelInput(inputFields));
+        model = new LocalModel(modelId,
+            definition,
+            new TrainedModelInput(inputFields),
+            Collections.singletonMap("field.foo", "field.foo.keyword"));
         result = getSingleValue(model, fields, new ClassificationConfig(0));
         assertThat(result.value(), equalTo(0.0));
         assertThat(result.valueAsString(), equalTo("not_to_be"));
@@ -90,11 +97,14 @@ public void testRegression() throws Exception {
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildRegression())
             .build();
-        Model model = new LocalModel("regression_model", trainedModelDefinition, new TrainedModelInput(inputFields));
+        Model model = new LocalModel("regression_model",
+            trainedModelDefinition,
+            new TrainedModelInput(inputFields),
+            Collections.singletonMap("bar", "bar.keyword"));
 
         Map<String, Object> fields = new HashMap<>() {{
-            put("field.foo", 1.0);
-            put("field.bar", 0.5);
+            put("foo", 1.0);
+            put("bar.keyword", 0.5);
             put("categorical", "dog");
         }};
 
@@ -114,7 +124,7 @@ public void testAllFieldsMissing() throws Exception {
             .setPreProcessors(Arrays.asList(new OneHotEncoding("categorical", oneHotMap())))
             .setTrainedModel(buildRegression())
             .build();
-        Model model = new LocalModel("regression_model", trainedModelDefinition, new TrainedModelInput(inputFields));
+        Model model = new LocalModel("regression_model", trainedModelDefinition, new TrainedModelInput(inputFields), null);
 
         Map<String, Object> fields = new HashMap<>() {{
             put("something", 1.0);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
index f5d3fc84b8eeb..585bfd8545678 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
@@ -10,16 +10,13 @@
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
-import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
 import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
 import org.junit.Before;
 
-import java.util.Collection;
 import java.util.List;
 
 public class AnnotationIndexIT extends MlSingleNodeTestCase {
@@ -34,11 +31,6 @@ protected Settings nodeSettings()  {
         return newSettings.build();
     }
 
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(LocalStateMachineLearning.class);
-    }
-
     @Before
     public void createComponents() throws Exception {
         waitForMlTemplates();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
index 7f783f45bc774..23fe7704a5fd3 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.ModelPlot;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
 import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor;
@@ -93,7 +94,11 @@ public class AutodetectResultProcessorIT extends MlSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(LocalStateMachineLearning.class, ReindexPlugin.class);
+        return pluginList(
+            LocalStateMachineLearning.class,
+            ReindexPlugin.class,
+            // ILM is required for .ml-state template index settings
+            IndexLifecycle.class);
     }
 
     @Before
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
index 899be32a217cb..23275958b3c10 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
@@ -165,7 +165,7 @@ public void testJobAutoClose() throws Exception {
         indexRequest = new IndexRequest("data");
         indexRequest.source("time", 1407083600L);
         client().index(indexRequest).get();
-        refresh("*", ".ml-*");
+        refresh("*");
 
         Job.Builder job = createScheduledJob("job_id");
         PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
index 92e18a6e6430f..996ffec02af52 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
@@ -6,14 +6,17 @@
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -37,6 +40,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
 import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
+import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
@@ -70,8 +74,6 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
@@ -82,6 +84,9 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
 import static org.elasticsearch.mock.orig.Mockito.doAnswer;
 import static org.elasticsearch.mock.orig.Mockito.doReturn;
 import static org.elasticsearch.mock.orig.Mockito.doThrow;
@@ -158,10 +163,21 @@ public void setup() throws Exception {
                 new HashSet<>(Arrays.asList(MachineLearning.MAX_OPEN_JOBS_PER_NODE,
                     ResultsPersisterService.PERSIST_RESULTS_MAX_RETRIES)));
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
-        MetaData metaData = mock(MetaData.class);
-        SortedMap<String, AliasOrIndex> aliasOrIndexSortedMap = new TreeMap<>();
-        aliasOrIndexSortedMap.put(AnomalyDetectorsIndex.jobStateIndexWriteAlias(), mock(AliasOrIndex.Alias.class));
-        when(metaData.getAliasAndIndexLookup()).thenReturn(aliasOrIndexSortedMap);
+        MetaData metaData = MetaData.builder()
+            .indices(ImmutableOpenMap.<String, IndexMetaData>builder()
+                .fPut(
+                    AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-000001",
+                    IndexMetaData.builder(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-000001")
+                        .settings(
+                            Settings.builder()
+                                .put(SETTING_NUMBER_OF_SHARDS, 1)
+                                .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                                .put(SETTING_VERSION_CREATED, Version.CURRENT)
+                                .build())
+                        .putAlias(AliasMetaData.builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).build())
+                        .build())
+                .build())
+            .build();
         clusterState = mock(ClusterState.class);
         when(clusterState.getMetaData()).thenReturn(metaData);
         when(clusterState.metaData()).thenReturn(metaData);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
index 599dd2ab62204..887cd17eb6b50 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.test.MockHttpTransport;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
@@ -54,6 +55,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.junit.After;
@@ -92,13 +94,18 @@ protected Settings nodeSettings(int nodeOrdinal) {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
+        settings.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
         return settings.build();
     }
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LocalStateMachineLearning.class, CommonAnalysisPlugin.class,
-                ReindexPlugin.class);
+        return Arrays.asList(
+            LocalStateMachineLearning.class,
+            CommonAnalysisPlugin.class,
+            ReindexPlugin.class,
+            // ILM is required for .ml-state template index settings
+            IndexLifecycle.class);
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
index cbb50605f72e2..3439626887ac0 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
@@ -28,6 +28,7 @@
 import java.util.Objects;
 import java.util.StringJoiner;
 
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT;
@@ -213,6 +214,7 @@ protected Object unwrapMultiValue(Object values) {
     protected boolean isFromDocValuesOnly(DataType dataType) {
         return dataType == KEYWORD // because of ignore_above.
                     || dataType == DATETIME
+                    || dataType == CONSTANT_KEYWORD // because a non-existent value is considered the constant value itself
                     || dataType == SCALED_FLOAT; // because of scaling_factor
     }
     
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
index dbe751276be0d..dfde94b5df76e 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
@@ -6,8 +6,6 @@
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
-import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 
@@ -26,15 +24,4 @@ protected NodeInfo<Like> info() {
     protected Like replaceChild(Expression newLeft) {
         return new Like(source(), newLeft, pattern());
     }
-
-    @Override
-    public Boolean fold() {
-        Object val = field().fold();
-        return RegexOperation.match(val, pattern().asJavaRegex());
-    }
-
-    @Override
-    protected Processor makeProcessor() {
-        return new RegexProcessor(pattern().asJavaRegex());
-    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
index fd6663b4b5ac3..096f7d8ae27e1 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
@@ -17,7 +17,7 @@
  *
  * To prevent conflicts with ES, the string and char must be validated to not contain '*'.
  */
-public class LikePattern {
+public class LikePattern implements StringPattern {
 
     private final String pattern;
     private final char escape;
@@ -43,9 +43,7 @@ public char escape() {
         return escape;
     }
 
-    /**
-     * Returns the pattern in (Java) regex format.
-     */
+    @Override
     public String asJavaRegex() {
         return regex;
     }
@@ -83,4 +81,4 @@ public boolean equals(Object obj) {
         return Objects.equals(pattern, other.pattern)
                 && escape == other.escape;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
index 9ec70bc79d1d8..af40f4ffb1750 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
@@ -6,14 +6,12 @@
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
-import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 
-public class RLike extends RegexMatch<String> {
+public class RLike extends RegexMatch<RLikePattern> {
 
-    public RLike(Source source, Expression value, String pattern) {
+    public RLike(Source source, Expression value, RLikePattern pattern) {
         super(source, value, pattern);
     }
 
@@ -26,15 +24,4 @@ protected NodeInfo<RLike> info() {
     protected RLike replaceChild(Expression newChild) {
         return new RLike(source(), newChild, pattern());
     }
-    
-    @Override
-    public Boolean fold() {
-        Object val = field().fold();
-        return RegexOperation.match(val, pattern());
-    }
-
-    @Override
-    protected Processor makeProcessor() {
-        return new RegexProcessor(pattern());
-    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
new file mode 100644
index 0000000000000..69747f03e2150
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ql.expression.predicate.regex;
+
+public class RLikePattern implements StringPattern {
+
+    private final String regexpPattern;
+
+    public RLikePattern(String regexpPattern) {
+        this.regexpPattern = regexpPattern;
+    }
+
+    @Override
+    public String asJavaRegex() {
+        return regexpPattern;
+    }
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
index 517c243992c24..565a75d489765 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
@@ -10,15 +10,19 @@
 import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.Nullability;
 import org.elasticsearch.xpack.ql.expression.function.scalar.UnaryScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.Objects;
 
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
 
-public abstract class RegexMatch<T> extends UnaryScalarFunction {
+public abstract class RegexMatch<T extends StringPattern> extends UnaryScalarFunction {
 
     private final T pattern;
     
@@ -54,8 +58,30 @@ public boolean foldable() {
         // right() is not directly foldable in any context but Like can fold it.
         return field().foldable();
     }
-    
+
+    @Override
+    public Boolean fold() {
+        Object val = field().fold();
+        return RegexProcessor.RegexOperation.match(val, pattern().asJavaRegex());
+    }
+
     @Override
+    protected Processor makeProcessor() {
+        return new RegexProcessor(pattern().asJavaRegex());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate fieldAsScript = asScript(field());
+        return new ScriptTemplate(
+                formatTemplate(format("{sql}.", "regex({},{})", fieldAsScript.template())),
+                paramsBuilder()
+                        .script(fieldAsScript.params())
+                        .variable(pattern.asJavaRegex())
+                        .build(),
+                dataType());
+    }
+
     public boolean equals(Object obj) {
         return super.equals(obj) && Objects.equals(((RegexMatch<?>) obj).pattern(), pattern());
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
new file mode 100644
index 0000000000000..6dac865b4ac5b
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ql.expression.predicate.regex;
+
+interface StringPattern {
+    /**
+     * Returns the pattern in (Java) regex format.
+     */
+    String asJavaRegex();
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
index 5d003dd80bf08..2b8cddfcf8b15 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypeRegistry;
 import org.elasticsearch.xpack.ql.type.DateEsField;
@@ -60,6 +61,7 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.action.ActionListener.wrap;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
@@ -298,8 +300,13 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
             StringBuilder errorMessage = new StringBuilder();
 
             boolean hasUnmapped = types.containsKey(UNMAPPED);
+            // a keyword field and a constant_keyword field with the same name in two different indices are considered "compatible"
+            // since a common use case of constant_keyword field involves two indices with a field having the same name: one being
+            // a keyword, the other being a constant_keyword
+            boolean hasCompatibleKeywords = types.containsKey(KEYWORD.esType()) && types.containsKey(CONSTANT_KEYWORD.esType());
+            int allowedTypesCount = (hasUnmapped ? 2 : 1) + (hasCompatibleKeywords ? 1 : 0);
 
-            if (types.size() > (hasUnmapped ? 2 : 1)) {
+            if (types.size() > allowedTypesCount) {
                 // build the error message
                 // and create a MultiTypeField
 
@@ -344,6 +351,11 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
                 }
             }
 
+            // if there are both a keyword and a constant_keyword type for this field, only keep the keyword as a common compatible type
+            if (hasCompatibleKeywords) {
+                types.remove(CONSTANT_KEYWORD.esType());
+            }
+
             // everything checks
             return null;
         });
@@ -435,6 +447,9 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa
         if (esType == DATETIME) {
             return new DateEsField(fieldName, props, isAggregateable);
         }
+        if (esType == CONSTANT_KEYWORD) {
+            return new ConstantKeywordEsField(fieldName);
+        }
         if (esType == UNSUPPORTED) {
             return new UnsupportedEsField(fieldName, typeName, null, props);
         }
@@ -501,14 +516,14 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
 
         for (Entry<String, Map<String, FieldCapabilities>> entry : sortedFields) {
             String fieldName = entry.getKey();
-            Map<String, FieldCapabilities> types = entry.getValue();
 
             // ignore size added by the mapper plugin
             if (FIELD_NAMES_BLACKLIST.contains(fieldName)) {
                 continue;
             }
 
-            // apply verification
+            Map<String, FieldCapabilities> types = new LinkedHashMap<>(entry.getValue());
+            // apply verification and possibly remove the "duplicate" CONSTANT_KEYWORD field type
             final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types);
 
             // filter meta fields and unmapped
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
index ebbd84ef07c7c..042e886f5d0fb 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
@@ -111,7 +111,7 @@ public static Query doTranslate(RegexMatch e, TranslatorHandler handler) {
             }
 
             if (e instanceof RLike) {
-                String pattern = ((RLike) e).pattern();
+                String pattern = ((RLike) e).pattern().asJavaRegex();
                 q = new RegexQuery(e.source(), targetFieldName, pattern);
             }
 
@@ -351,4 +351,4 @@ private static Query boolQuery(Source source, Query left, Query right, boolean i
         }
         return new BoolQuery(source, isAnd, left, right);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
new file mode 100644
index 0000000000000..9983599dc4def
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ql.type;
+
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
+
+/**
+ * SQL-related information about an index field with a constant_keyword type
+ */
+public class ConstantKeywordEsField extends KeywordEsField {
+
+    public ConstantKeywordEsField(String name) {
+        super(name, CONSTANT_KEYWORD, Collections.emptyMap(), true, Short.MAX_VALUE, false, false);
+    }
+
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
index 5400659c75ab4..a325f9287355e 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
@@ -21,6 +21,7 @@
 
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -59,9 +60,12 @@ public static DataType commonType(DataType left, DataType right) {
             return left;
         }
         if (isString(left) && isString(right)) {
-            if (left == TEXT) {
+            if (left == TEXT || right == TEXT) {
                 return TEXT;
             }
+            if (left == KEYWORD) {
+                return KEYWORD;
+            }
             return right;
         }
         if (left.isNumeric() && right.isNumeric()) {
@@ -120,7 +124,7 @@ public static Converter converterFor(DataType from, DataType to) {
             return DefaultConverter.TO_NULL;
         }
         // proper converters
-        if (to == KEYWORD || to == TEXT) {
+        if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
             return conversionToString(from);
         }
         if (to == LONG) {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
index 6253a5f89d7e2..bc04a3b0213a9 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
@@ -18,33 +18,34 @@
 public final class DataTypes {
 
     // @formatter:off
-    public static final DataType UNSUPPORTED  = new DataType("UNSUPPORTED", null, 0,                 false, false, false);
+    public static final DataType UNSUPPORTED      = new DataType("UNSUPPORTED", null, 0,                 false, false, false);
 
-    public static final DataType NULL         = new DataType("null",              0,                 false, false, false);
+    public static final DataType NULL             = new DataType("null",              0,                 false, false, false);
 
-    public static final DataType BOOLEAN      = new DataType("boolean",           1,                 false, false, false);
+    public static final DataType BOOLEAN          = new DataType("boolean",           1,                 false, false, false);
     // integer numeric
-    public static final DataType BYTE         = new DataType("byte",              Byte.BYTES,        true, false, true);
-    public static final DataType SHORT        = new DataType("short",             Short.BYTES,       true, false, true);
-    public static final DataType INTEGER      = new DataType("integer",           Integer.BYTES,     true, false, true);
-    public static final DataType LONG         = new DataType("long",              Long.BYTES,        true, false, true);
+    public static final DataType BYTE             = new DataType("byte",              Byte.BYTES,        true, false, true);
+    public static final DataType SHORT            = new DataType("short",             Short.BYTES,       true, false, true);
+    public static final DataType INTEGER          = new DataType("integer",           Integer.BYTES,     true, false, true);
+    public static final DataType LONG             = new DataType("long",              Long.BYTES,        true, false, true);
     // decimal numeric
-    public static final DataType DOUBLE       = new DataType("double",            Double.BYTES,      false, true, true);
-    public static final DataType FLOAT        = new DataType("float",             Float.BYTES,       false, true, true);
-    public static final DataType HALF_FLOAT   = new DataType("half_float",        Float.BYTES,       false, true, true);
-    public static final DataType SCALED_FLOAT = new DataType("scaled_float",      Long.BYTES,        false, true, true);
+    public static final DataType DOUBLE           = new DataType("double",            Double.BYTES,      false, true, true);
+    public static final DataType FLOAT            = new DataType("float",             Float.BYTES,       false, true, true);
+    public static final DataType HALF_FLOAT       = new DataType("half_float",        Float.BYTES,       false, true, true);
+    public static final DataType SCALED_FLOAT     = new DataType("scaled_float",      Long.BYTES,        false, true, true);
     // string
-    public static final DataType KEYWORD      = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
-    public static final DataType TEXT         = new DataType("text",              Integer.MAX_VALUE, false, false, false);
+    public static final DataType KEYWORD          = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
+    public static final DataType TEXT             = new DataType("text",              Integer.MAX_VALUE, false, false, false);
+    public static final DataType CONSTANT_KEYWORD = new DataType("constant_keyword",  Integer.MAX_VALUE, false, false, true);
     // date
-    public static final DataType DATETIME     = new DataType("DATETIME", "date",  Long.BYTES,        false, false, true);
+    public static final DataType DATETIME         = new DataType("DATETIME", "date",  Long.BYTES,        false, false, true);
     // ip
     public static final DataType IP           = new DataType("ip",                45,                false, false, true);
     // binary
-    public static final DataType BINARY       = new DataType("binary",            Integer.MAX_VALUE, false, false, true);
+    public static final DataType BINARY           = new DataType("binary",            Integer.MAX_VALUE, false, false, true);
     // complex types
-    public static final DataType OBJECT       = new DataType("object",            0,                 false, false, false);
-    public static final DataType NESTED       = new DataType("nested",            0,                 false, false, false);
+    public static final DataType OBJECT           = new DataType("object",            0,                 false, false, false);
+    public static final DataType NESTED           = new DataType("nested",            0,                 false, false, false);
     //@formatter:on
     
     private static final Collection<DataType> TYPES = Arrays.asList(
@@ -61,6 +62,7 @@ public final class DataTypes {
             SCALED_FLOAT,
             KEYWORD,
             TEXT,
+            CONSTANT_KEYWORD,
             DATETIME,
             IP,
             BINARY,
@@ -132,7 +134,7 @@ public static boolean isUnsupported(DataType from) {
     }
 
     public static boolean isString(DataType t) {
-        return t == KEYWORD || t == TEXT;
+        return t == KEYWORD || t == TEXT || t == CONSTANT_KEYWORD;
     }
 
     public static boolean isPrimitive(DataType t) {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
index 78d84946da4c9..65c679a5ad034 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
@@ -29,7 +29,12 @@ public KeywordEsField(String name, Map<String, EsField> properties, boolean hasD
     
     public KeywordEsField(String name, Map<String, EsField> properties, boolean hasDocValues, int precision,
             boolean normalized, boolean isAlias) {
-        super(name, KEYWORD, properties, hasDocValues, isAlias);
+        this(name, KEYWORD, properties, hasDocValues, precision, normalized, isAlias);
+    }
+
+    protected KeywordEsField(String name, DataType esDataType, Map<String, EsField> properties, boolean hasDocValues, int precision,
+            boolean normalized, boolean isAlias) {
+        super(name, esDataType, properties, hasDocValues, isAlias);
         this.precision = precision;
         this.normalized = normalized;
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
index e3e325be23f33..777fa9baea8c6 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
@@ -11,6 +11,7 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
 
@@ -44,7 +45,7 @@ public Exact getExactInfo() {
     private Tuple<EsField, String> findExact() {
         EsField field = null;
         for (EsField property : getProperties().values()) {
-            if (property.getDataType() == KEYWORD && property.getExactInfo().hasExact()) {
+            if ((property.getDataType() == KEYWORD || property.getDataType() == CONSTANT_KEYWORD) && property.getExactInfo().hasExact()) {
                 if (field != null) {
                     return new Tuple<>(null, "Multiple exact keyword candidates available for [" + getName() +
                         "]; specify which one to use");
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
index 44202b006fcf6..136d6e8155451 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
@@ -14,6 +14,7 @@
 import java.util.Map.Entry;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED;
@@ -89,6 +90,8 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
                 int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
                 boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
                 field = new KeywordEsField(name, properties, docValues, length, normalized);
+            } else if (esDataType == CONSTANT_KEYWORD) {
+                field = new ConstantKeywordEsField(name);
             } else if (esDataType == DATETIME) {
                 field = new DateEsField(name, properties, docValues);
             } else if (esDataType == UNSUPPORTED) {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
index abc9fee4d97db..b40edfc381fc5 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
@@ -20,6 +20,7 @@
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
@@ -53,7 +54,7 @@ static class ValueAndCompatibleTypes {
             new ValueAndCompatibleTypes(ESTestCase::randomLong, LONG, FLOAT, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomFloat, FLOAT, LONG, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomDouble, DOUBLE, LONG, FLOAT, BOOLEAN),
-            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD));
+            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD, CONSTANT_KEYWORD));
 
     public static Literal randomLiteral() {
         ValueAndCompatibleTypes gen = randomFrom(GENERATORS);
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
index b853443a0ea7c..9121b42e7c9c8 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
@@ -48,13 +49,13 @@
 import java.util.List;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.ql.TestUtils.of;
 import static org.elasticsearch.xpack.ql.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.ql.expression.Literal.NULL;
 import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.ql.TestUtils.of;
 
 public class OptimizerRulesTests extends ESTestCase {
 
@@ -189,7 +190,7 @@ public void testConstantFoldingLikes() {
                 new ConstantFolding().rule(new Like(EMPTY, of("test_emp"), new LikePattern("test%", (char) 0)))
                         .canonical());
         assertEquals(TRUE,
-                new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), "test.emp")).canonical());
+                new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
     }
 
     public void testArithmeticFolding() {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
index f1188ed546e5a..0010e99fdf510 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
@@ -17,6 +17,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -361,6 +362,7 @@ public void testCommonType() {
         assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
         assertEquals(NULL, commonType(NULL, NULL));
         assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
+        assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
         assertEquals(LONG, commonType(TEXT, LONG));
         assertEquals(SHORT, commonType(SHORT, BYTE));
         assertEquals(FLOAT, commonType(BYTE, FLOAT));
@@ -370,6 +372,11 @@ public void testCommonType() {
         // strings
         assertEquals(TEXT, commonType(TEXT, KEYWORD));
         assertEquals(TEXT, commonType(KEYWORD, TEXT));
+        assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
+        assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
+        assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
+        assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
+        assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
     }
 
     public void testEsDataTypes() {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
index 9958d00b02929..7adbf9306d7b8 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
@@ -14,6 +14,7 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
@@ -135,9 +136,10 @@ public void testMultiField() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field.getDataType(), is(TEXT));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(2));
+        assertThat(fields.size(), is(3));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
+        assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
     }
 
     public void testMultiFieldTooManyOptions() {
@@ -148,9 +150,10 @@ public void testMultiFieldTooManyOptions() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field, instanceOf(TextEsField.class));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(2));
+        assertThat(fields.size(), is(3));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
+        assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
     }
 
     public void testNestedDoc() {
@@ -173,6 +176,13 @@ public void testIpField() {
         assertThat(dt.getDataType().typeName(), is("ip"));
     }
 
+    public void testConstantKeywordField() {
+        Map<String, EsField> mapping = loadMapping("mapping-constant-keyword.json");
+        assertThat(mapping.size(), is(1));
+        EsField dt = mapping.get("full_name");
+        assertThat(dt.getDataType().typeName(), is("constant_keyword"));
+    }
+
     public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json b/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
new file mode 100644
index 0000000000000..7f248dc26fba6
--- /dev/null
+++ b/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
@@ -0,0 +1,8 @@
+{
+    "properties" : {
+        "full_name" : {
+            "type" : "constant_keyword",
+            "value" : "foo"
+        }
+    }
+}
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
index 9e293d4271333..8adc4c64e8476 100644
--- a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
+++ b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
@@ -9,6 +9,10 @@
                 "english" : {
                     "type" : "text",
                     "analyzer" : "english"
+                },
+                "constant" : {
+                    "type" : "constant_keyword",
+                    "value" : "some constant value"
                 }
             }
         }
diff --git a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java
new file mode 100644
index 0000000000000..7fcec7d6cae74
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public abstract class SecurityInBasicRestTestCase extends ESRestTestCase {
+    private RestHighLevelClient highLevelAdminClient;
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("security_test_user", new SecureString("security-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    private RestHighLevelClient getHighLevelAdminClient() {
+        if (highLevelAdminClient == null) {
+            highLevelAdminClient = new RestHighLevelClient(
+                adminClient(),
+                ignore -> {
+                },
+                List.of()) {
+            };
+        }
+        return highLevelAdminClient;
+    }
+}
diff --git a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
index 837421f6000d5..98f92c750951c 100644
--- a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
@@ -11,10 +11,6 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
 import org.elasticsearch.xpack.security.authc.InternalRealms;
 
@@ -24,29 +20,12 @@
 import java.util.Base64;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SecurityWithBasicLicenseIT extends ESRestTestCase {
-
-    @Override
-    protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("security_test_user", new SecureString("security-test-password".toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
+public class SecurityWithBasicLicenseIT extends SecurityInBasicRestTestCase {
 
     public void testWithBasicLicense() throws Exception {
         checkLicenseType("basic");
diff --git a/x-pack/plugin/security/qa/security-trial/build.gradle b/x-pack/plugin/security/qa/security-trial/build.gradle
new file mode 100644
index 0000000000000..e8e0b547c1828
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-trial/build.gradle
@@ -0,0 +1,28 @@
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 2
+
+  setting 'xpack.ilm.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.ssl.diagnose.trust', 'true'
+  setting 'xpack.security.http.ssl.enabled', 'false'
+  setting 'xpack.security.transport.ssl.enabled', 'false'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
+
+  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+  user username: "admin_user", password: "admin-password"
+  user username: "security_test_user", password: "security-test-password", role: "security_test_role"
+}
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
new file mode 100644
index 0000000000000..2fe99bc8ad48d
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.security.CreateTokenRequest;
+import org.elasticsearch.client.security.CreateTokenResponse;
+import org.elasticsearch.client.security.DeleteRoleRequest;
+import org.elasticsearch.client.security.DeleteUserRequest;
+import org.elasticsearch.client.security.GetApiKeyRequest;
+import org.elasticsearch.client.security.GetApiKeyResponse;
+import org.elasticsearch.client.security.InvalidateApiKeyRequest;
+import org.elasticsearch.client.security.PutRoleRequest;
+import org.elasticsearch.client.security.PutUserRequest;
+import org.elasticsearch.client.security.RefreshPolicy;
+import org.elasticsearch.client.security.support.ApiKey;
+import org.elasticsearch.client.security.user.User;
+import org.elasticsearch.client.security.user.privileges.Role;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase {
+    private RestHighLevelClient highLevelAdminClient;
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("security_test_user", new SecureString("security-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    protected void createUser(String username, SecureString password, List<String> roles) throws IOException {
+        final RestHighLevelClient client = getHighLevelAdminClient();
+        client.security().putUser(PutUserRequest.withPassword(new User(username, roles), password.getChars(), true,
+            RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT);
+    }
+
+    protected void createRole(String name, Collection<String> clusterPrivileges) throws IOException {
+        final RestHighLevelClient client = getHighLevelAdminClient();
+        final Role role = Role.builder().name(name).clusterPrivileges(clusterPrivileges).build();
+        client.security().putRole(new PutRoleRequest(role, null), RequestOptions.DEFAULT);
+    }
+
+    /**
+     * @return A tuple of (access-token, refresh-token)
+     */
+    protected Tuple<String, String> createOAuthToken(String username, SecureString password) throws IOException {
+        final RestHighLevelClient client = getHighLevelAdminClient();
+        final CreateTokenRequest request = CreateTokenRequest.passwordGrant(username, password.getChars());
+        final CreateTokenResponse response = client.security().createToken(request, RequestOptions.DEFAULT);
+        return new Tuple(response.getAccessToken(), response.getRefreshToken());
+    }
+
+    protected void deleteUser(String username) throws IOException {
+        final RestHighLevelClient client = getHighLevelAdminClient();
+        client.security().deleteUser(new DeleteUserRequest(username), RequestOptions.DEFAULT);
+    }
+
+    protected void deleteRole(String name) throws IOException {
+        final RestHighLevelClient client = getHighLevelAdminClient();
+        client.security().deleteRole(new DeleteRoleRequest(name), RequestOptions.DEFAULT);
+    }
+
+    protected void invalidateApiKeysForUser(String username) throws IOException {
+        final RestHighLevelClient client = getHighLevelAdminClient();
+        client.security().invalidateApiKey(InvalidateApiKeyRequest.usingUserName(username), RequestOptions.DEFAULT);
+    }
+
+    protected ApiKey getApiKey(String id) throws IOException {
+        final RestHighLevelClient client = getHighLevelAdminClient();
+        final GetApiKeyResponse response = client.security().getApiKey(GetApiKeyRequest.usingApiKeyId(id, false), RequestOptions.DEFAULT);
+        assertThat(response.getApiKeyInfos(), Matchers.iterableWithSize(1));
+        return response.getApiKeyInfos().get(0);
+    }
+
+    private RestHighLevelClient getHighLevelAdminClient() {
+        if (highLevelAdminClient == null) {
+            highLevelAdminClient = new RestHighLevelClient(
+                adminClient(),
+                ignore -> {
+                },
+                List.of()) {
+            };
+        }
+        return highLevelAdminClient;
+    }
+}
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
new file mode 100644
index 0000000000000..daa0cfc303142
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.apikey;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.security.support.ApiKey;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * Integration Rest Tests relating to API Keys.
+ * Tested against a trial license
+ */
+public class ApiKeyRestIT extends SecurityOnTrialLicenseRestTestCase {
+
+    private static final String SYSTEM_USER = "system_user";
+    private static final SecureString SYSTEM_USER_PASSWORD = new SecureString("sys-pass".toCharArray());
+    private static final String END_USER = "end_user";
+    private static final SecureString END_USER_PASSWORD = new SecureString("user-pass".toCharArray());
+
+    @Before
+    public void createUsers() throws IOException {
+        createUser(SYSTEM_USER, SYSTEM_USER_PASSWORD, List.of("system_role"));
+        createRole("system_role", Set.of("manage_api_key"));
+        createUser(END_USER, END_USER_PASSWORD, List.of("user_role"));
+        createRole("user_role", Set.of("monitor"));
+    }
+
+    @After
+    public void cleanUp() throws IOException {
+        deleteUser("system_user");
+        deleteUser("end_user");
+        deleteRole("system_role");
+        deleteRole("user_role");
+        invalidateApiKeysForUser(END_USER);
+    }
+
+    public void testGrantApiKeyForOtherUserWithPassword() throws IOException {
+        Request request = new Request("POST", "_security/api_key/grant");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+            UsernamePasswordToken.basicAuthHeaderValue(SYSTEM_USER, SYSTEM_USER_PASSWORD)));
+        final Map<String, Object> requestBody = Map.ofEntries(
+            Map.entry("grant_type", "password"),
+            Map.entry("username", END_USER),
+            Map.entry("password", END_USER_PASSWORD.toString()),
+            Map.entry("api_key", Map.of("name", "test_api_key_password"))
+        );
+        request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString());
+
+        final Response response = client().performRequest(request);
+        final Map<String, Object> responseBody = entityAsMap(response);
+
+        assertThat(responseBody.get("name"), equalTo("test_api_key_password"));
+        assertThat(responseBody.get("id"), notNullValue());
+        assertThat(responseBody.get("id"), instanceOf(String.class));
+
+        ApiKey apiKey = getApiKey((String) responseBody.get("id"));
+        assertThat(apiKey.getUsername(), equalTo(END_USER));
+    }
+
+    public void testGrantApiKeyForOtherUserWithAccessToken() throws IOException {
+        final Tuple<String, String> token = super.createOAuthToken(END_USER, END_USER_PASSWORD);
+        final String accessToken = token.v1();
+
+        final Request request = new Request("POST", "_security/api_key/grant");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+            UsernamePasswordToken.basicAuthHeaderValue(SYSTEM_USER, SYSTEM_USER_PASSWORD)));
+        final Map<String, Object> requestBody = Map.ofEntries(
+            Map.entry("grant_type", "access_token"),
+            Map.entry("access_token", accessToken),
+            Map.entry("api_key", Map.of("name", "test_api_key_token", "expiration", "2h"))
+        );
+        request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString());
+
+        final Instant before = Instant.now();
+        final Response response = client().performRequest(request);
+        final Instant after = Instant.now();
+        final Map<String, Object> responseBody = entityAsMap(response);
+
+        assertThat(responseBody.get("name"), equalTo("test_api_key_token"));
+        assertThat(responseBody.get("id"), notNullValue());
+        assertThat(responseBody.get("id"), instanceOf(String.class));
+
+        ApiKey apiKey = getApiKey((String) responseBody.get("id"));
+        assertThat(apiKey.getUsername(), equalTo(END_USER));
+
+        Instant minExpiry = before.plus(2, ChronoUnit.HOURS);
+        Instant maxExpiry = after.plus(2, ChronoUnit.HOURS);
+        assertThat(apiKey.getExpiration(), notNullValue());
+        assertThat(apiKey.getExpiration(), greaterThanOrEqualTo(minExpiry));
+        assertThat(apiKey.getExpiration(), lessThanOrEqualTo(maxExpiry));
+    }
+}
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml b/x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml
new file mode 100644
index 0000000000000..9b2171257fc61
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-trial/src/test/resources/roles.yml
@@ -0,0 +1,8 @@
+# A basic role that is used to test security
+security_test_role:
+  cluster:
+    - monitor
+    - "cluster:admin/xpack/license/*"
+  indices:
+    - names: [ "index_allowed" ]
+      privileges: [ "read", "write", "create_index" ]
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 885de69564c2b..1f62f6c341f87 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -81,6 +81,7 @@
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.DelegatePkiAuthenticationAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectAuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectLogoutAction;
@@ -142,6 +143,7 @@
 import org.elasticsearch.xpack.security.action.TransportCreateApiKeyAction;
 import org.elasticsearch.xpack.security.action.TransportDelegatePkiAuthenticationAction;
 import org.elasticsearch.xpack.security.action.TransportGetApiKeyAction;
+import org.elasticsearch.xpack.security.action.TransportGrantApiKeyAction;
 import org.elasticsearch.xpack.security.action.TransportInvalidateApiKeyAction;
 import org.elasticsearch.xpack.security.action.filter.SecurityActionFilter;
 import org.elasticsearch.xpack.security.action.oidc.TransportOpenIdConnectAuthenticateAction;
@@ -206,6 +208,7 @@
 import org.elasticsearch.xpack.security.rest.action.RestDelegatePkiAuthenticationAction;
 import org.elasticsearch.xpack.security.rest.action.apikey.RestCreateApiKeyAction;
 import org.elasticsearch.xpack.security.rest.action.apikey.RestGetApiKeyAction;
+import org.elasticsearch.xpack.security.rest.action.apikey.RestGrantApiKeyAction;
 import org.elasticsearch.xpack.security.rest.action.apikey.RestInvalidateApiKeyAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestInvalidateTokenAction;
@@ -753,6 +756,7 @@ public void onIndexModule(IndexModule module) {
                 new ActionHandler<>(PutPrivilegesAction.INSTANCE, TransportPutPrivilegesAction.class),
                 new ActionHandler<>(DeletePrivilegesAction.INSTANCE, TransportDeletePrivilegesAction.class),
                 new ActionHandler<>(CreateApiKeyAction.INSTANCE, TransportCreateApiKeyAction.class),
+                new ActionHandler<>(GrantApiKeyAction.INSTANCE, TransportGrantApiKeyAction.class),
                 new ActionHandler<>(InvalidateApiKeyAction.INSTANCE, TransportInvalidateApiKeyAction.class),
                 new ActionHandler<>(GetApiKeyAction.INSTANCE, TransportGetApiKeyAction.class),
                 new ActionHandler<>(DelegatePkiAuthenticationAction.INSTANCE, TransportDelegatePkiAuthenticationAction.class),
@@ -808,6 +812,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                 new RestPutPrivilegesAction(settings, getLicenseState()),
                 new RestDeletePrivilegesAction(settings, getLicenseState()),
                 new RestCreateApiKeyAction(settings, getLicenseState()),
+                new RestGrantApiKeyAction(settings, getLicenseState()),
                 new RestInvalidateApiKeyAction(settings, getLicenseState()),
                 new RestGetApiKeyAction(settings, getLicenseState()),
                 new RestDelegatePkiAuthenticationAction(settings, getLicenseState())
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
index 72a92516e59ed..730affc7ea953 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
@@ -6,12 +6,10 @@
 
 package org.elasticsearch.xpack.security.action;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
@@ -20,32 +18,24 @@
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
+import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 
-import java.util.Arrays;
-import java.util.HashSet;
-
 /**
  * Implementation of the action needed to create an API key
  */
 public final class TransportCreateApiKeyAction extends HandledTransportAction<CreateApiKeyRequest, CreateApiKeyResponse> {
 
-    private final ApiKeyService apiKeyService;
+    private final ApiKeyGenerator generator;
     private final SecurityContext securityContext;
-    private final CompositeRolesStore rolesStore;
-    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportCreateApiKeyAction(TransportService transportService, ActionFilters actionFilters, ApiKeyService apiKeyService,
                                        SecurityContext context, CompositeRolesStore rolesStore, NamedXContentRegistry xContentRegistry) {
-        super(CreateApiKeyAction.NAME, transportService, actionFilters, (Writeable.Reader<CreateApiKeyRequest>) CreateApiKeyRequest::new);
-        this.apiKeyService = apiKeyService;
+        super(CreateApiKeyAction.NAME, transportService, actionFilters, CreateApiKeyRequest::new);
+        this.generator = new ApiKeyGenerator(apiKeyService, rolesStore, xContentRegistry);
         this.securityContext = context;
-        this.rolesStore = rolesStore;
-        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
@@ -54,19 +44,7 @@ protected void doExecute(Task task, CreateApiKeyRequest request, ActionListener<
         if (authentication == null) {
             listener.onFailure(new IllegalStateException("authentication is required"));
         } else {
-            rolesStore.getRoleDescriptors(new HashSet<>(Arrays.asList(authentication.getUser().roles())),
-                ActionListener.wrap(roleDescriptors -> {
-                        for (RoleDescriptor rd : roleDescriptors) {
-                            try {
-                                DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry);
-                            } catch (ElasticsearchException | IllegalArgumentException e) {
-                                listener.onFailure(e);
-                                return;
-                            }
-                        }
-                        apiKeyService.createApiKey(authentication, request, roleDescriptors, listener);
-                    },
-                    listener::onFailure));
+            generator.generateApiKey(authentication, request, listener);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java
new file mode 100644
index 0000000000000..d41754266329c
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyAction.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.action;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportMessage;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyRequest;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
+import org.elasticsearch.xpack.security.authc.AuthenticationService;
+import org.elasticsearch.xpack.security.authc.TokenService;
+import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+
+/**
+ * Implementation of the action needed to create an API key on behalf of another user (using an OAuth style "grant")
+ */
+public final class TransportGrantApiKeyAction extends HandledTransportAction<GrantApiKeyRequest, CreateApiKeyResponse> {
+
+    private final ThreadContext threadContext;
+    private final ApiKeyGenerator generator;
+    private final AuthenticationService authenticationService;
+    private final TokenService tokenService;
+
+    @Inject
+    public TransportGrantApiKeyAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool,
+                                      ApiKeyService apiKeyService, AuthenticationService authenticationService, TokenService tokenService,
+                                      CompositeRolesStore rolesStore, NamedXContentRegistry xContentRegistry) {
+        this(transportService, actionFilters, threadPool.getThreadContext(),
+            new ApiKeyGenerator(apiKeyService, rolesStore, xContentRegistry), authenticationService, tokenService
+        );
+    }
+
+    // Constructor for testing
+    TransportGrantApiKeyAction(TransportService transportService, ActionFilters actionFilters, ThreadContext threadContext,
+                                      ApiKeyGenerator generator, AuthenticationService authenticationService, TokenService tokenService) {
+        super(GrantApiKeyAction.NAME, transportService, actionFilters, GrantApiKeyRequest::new);
+        this.threadContext = threadContext;
+        this.generator = generator;
+        this.authenticationService = authenticationService;
+        this.tokenService = tokenService;
+    }
+
+    @Override
+    protected void doExecute(Task task, GrantApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            resolveAuthentication(request.getGrant(), request, ActionListener.wrap(
+                authentication -> generator.generateApiKey(authentication, request.getApiKeyRequest(), listener),
+                listener::onFailure
+            ));
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
+    }
+
+    private void resolveAuthentication(GrantApiKeyRequest.Grant grant, TransportMessage message, ActionListener<Authentication> listener) {
+        switch (grant.getType()) {
+            case GrantApiKeyRequest.PASSWORD_GRANT_TYPE:
+                final UsernamePasswordToken token = new UsernamePasswordToken(grant.getUsername(), grant.getPassword());
+                authenticationService.authenticate(super.actionName, message, token, listener);
+                return;
+            case GrantApiKeyRequest.ACCESS_TOKEN_GRANT_TYPE:
+                tokenService.authenticateToken(grant.getAccessToken(), listener);
+                return;
+            default:
+                listener.onFailure(new ElasticsearchSecurityException("the grant type [{}] is not supported", grant.getType()));
+                return;
+        }
+    }
+
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index 5cd9c062914bc..021984d073a23 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -574,7 +574,7 @@ private boolean isEnabled() {
         return enabled && licenseState.isApiKeyServiceAllowed();
     }
 
-    private void ensureEnabled() {
+    public void ensureEnabled() {
         if (licenseState.isApiKeyServiceAllowed() == false) {
             throw LicenseUtils.newComplianceException("api keys");
         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index 24b40ba2922ae..c12b14477b8e7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -396,6 +396,35 @@ void getAndValidateToken(ThreadContext ctx, ActionListener<UserToken> listener)
         }
     }
 
+    /**
+     * Decodes the provided token, and validates it (for format, expiry and invalidation).
+     * If valid, the token's {@link Authentication} (see {@link UserToken#getAuthentication()} is provided to the listener.
+     * If the token is invalid (expired etc), then {@link ActionListener#onFailure(Exception)} will be called.
+     * If tokens are not enabled, or the token does not exist, {@link ActionListener#onResponse} will be called with a
+     * {@code null} authentication object.
+     */
+    public void authenticateToken(SecureString tokenString, ActionListener<Authentication> listener) {
+        ensureEnabled();
+        decodeToken(tokenString.toString(), ActionListener.wrap(userToken -> {
+            if (userToken != null) {
+                checkIfTokenIsValid(userToken, ActionListener.wrap(
+                    token -> {
+                        if (token == null) {
+                            // Typically this means that the index is unavailable, so _probably_ the token is invalid but the only
+                            // this we can say for certain is that we couldn't validate it. The logs will be more explicit.
+                            listener.onFailure(new IllegalArgumentException("Cannot validate access token"));
+                        } else {
+                            listener.onResponse(token.getAuthentication());
+                        }
+                    },
+                    listener::onFailure
+                ));
+            } else {
+                listener.onFailure(new IllegalArgumentException("Cannot decode access token"));
+            }
+        }, listener::onFailure));
+    }
+
     /**
      * Reads the authentication and metadata from the given token.
      * This method does not validate whether the token is expired or not.
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java
new file mode 100644
index 0000000000000..1e1e4c0339c3d
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.support;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+public class ApiKeyGenerator {
+
+    private final ApiKeyService apiKeyService;
+    private final CompositeRolesStore rolesStore;
+    private final NamedXContentRegistry xContentRegistry;
+
+    public ApiKeyGenerator(ApiKeyService apiKeyService, CompositeRolesStore rolesStore, NamedXContentRegistry xContentRegistry) {
+        this.apiKeyService = apiKeyService;
+        this.rolesStore = rolesStore;
+        this.xContentRegistry = xContentRegistry;
+    }
+
+    public void generateApiKey(Authentication authentication, CreateApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
+        if (authentication == null) {
+            listener.onFailure(new ElasticsearchSecurityException("no authentication available to generate API key"));
+            return;
+        }
+        apiKeyService.ensureEnabled();
+        rolesStore.getRoleDescriptors(new HashSet<>(Arrays.asList(authentication.getUser().roles())),
+            ActionListener.wrap(roleDescriptors -> {
+                    for (RoleDescriptor rd : roleDescriptors) {
+                        try {
+                            DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry);
+                        } catch (ElasticsearchException | IllegalArgumentException e) {
+                            listener.onFailure(e);
+                            return;
+                        }
+                    }
+                    apiKeyService.createApiKey(authentication, request, roleDescriptors, listener);
+                },
+                listener::onFailure));
+
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index b61716462637e..6113cf4dfd305 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -32,6 +32,9 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.transport.TransportActionProxy;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
@@ -239,17 +242,18 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
             // need to validate that the action is allowed and then move on
             authorizeIndexActionName(action, authorizationInfo, null, listener);
         } else if (request instanceof IndicesRequest == false && request instanceof IndicesAliasesRequest == false) {
-            // scroll is special
-            // some APIs are indices requests that are not actually associated with indices. For example,
-            // search scroll request, is categorized under the indices context, but doesn't hold indices names
-            // (in this case, the security check on the indices was done on the search request that initialized
-            // the scroll. Given that scroll is implemented using a context on the node holding the shard, we
-            // piggyback on it and enhance the context with the original authentication. This serves as our method
-            // to validate the scroll id only stays with the same user!
-            // note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any
-            // indices permission as it's categorized under cluster. This is why the scroll check is performed
-            // even before checking if the user has any indices permission.
             if (isScrollRelatedAction(action)) {
+                // scroll is special
+                // some APIs are indices requests that are not actually associated with indices. For example,
+                // search scroll request, is categorized under the indices context, but doesn't hold indices names
+                // (in this case, the security check on the indices was done on the search request that initialized
+                // the scroll. Given that scroll is implemented using a context on the node holding the shard, we
+                // piggyback on it and enhance the context with the original authentication. This serves as our method
+                // to validate the scroll id only stays with the same user!
+                // note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any
+                // indices permission as it's categorized under cluster. This is why the scroll check is performed
+                // even before checking if the user has any indices permission.
+
                 // if the action is a search scroll action, we first authorize that the user can execute the action for some
                 // index and if they cannot, we can fail the request early before we allow the execution of the action and in
                 // turn the shard actions
@@ -261,11 +265,22 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
                     // information such as the index and the incoming address of the request
                     listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
                 }
+            } else if (isAsyncSearchRelatedAction(action)) {
+                if (SubmitAsyncSearchAction.NAME.equals(action)) {
+                    // we check if the user has any indices permission when submitting an async-search request in order to be
+                    // able to fail the request early. Fine grained index-level permissions are handled by the search action
+                    // that is triggered internally by the submit API.
+                    authorizeIndexActionName(action, authorizationInfo, null, listener);
+                } else {
+                    // async-search actions other than submit have a custom security layer that checks if the current user is
+                    // the same as the user that submitted the original request so we can skip security here.
+                    listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
+                }
             } else {
-                assert false :
-                    "only scroll related requests are known indices api that don't support retrieving the indices they relate to";
-                listener.onFailure(new IllegalStateException("only scroll related requests are known indices api that don't support " +
-                    "retrieving the indices they relate to"));
+                assert false : "only scroll and async-search related requests are known indices api that don't " +
+                    "support retrieving the indices they relate to";
+                listener.onFailure(new IllegalStateException("only scroll and async-search related requests are known indices " +
+                    "api that don't support retrieving the indices they relate to"));
             }
         } else if (request instanceof IndicesRequest &&
             IndicesAndAliasesResolver.allowsRemoteIndices((IndicesRequest) request)) {
@@ -570,4 +585,10 @@ private static boolean isScrollRelatedAction(String action) {
             action.equals("indices:data/read/sql/close_cursor") ||
             action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
     }
+
+    private static boolean isAsyncSearchRelatedAction(String action) {
+        return action.equals(SubmitAsyncSearchAction.NAME) ||
+            action.equals(GetAsyncSearchAction.NAME) ||
+            action.equals(DeleteAsyncSearchAction.NAME);
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index adfe0d4dd8e4c..d898003a50147 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -78,7 +79,7 @@ public void validateSearchContext(SearchContext searchContext, TransportRequest
      * (or lookup) realm. To work around this we compare the username and the originating realm type.
      */
     static void ensureAuthenticatedUserIsSame(Authentication original, Authentication current, AuditTrailService auditTrailService,
-                                              long id, String action, TransportRequest request, String requestId,
+                                              SearchContextId id, String action, TransportRequest request, String requestId,
                                               AuthorizationInfo authorizationInfo) {
         // this is really a best effort attempt since we cannot guarantee principal uniqueness
         // and realm names can change between nodes.
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java
new file mode 100644
index 0000000000000..0ce0c6588e810
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.rest.action.apikey;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyRequest;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+/**
+ * Rest action to create an API key on behalf of another user. Loosely mimics the API of
+ * {@link org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction} combined with {@link RestCreateApiKeyAction}
+ */
+public final class RestGrantApiKeyAction extends ApiKeyBaseRestHandler {
+
+    static final ObjectParser<GrantApiKeyRequest, Void> PARSER = new ObjectParser<>("grant_api_key_request", GrantApiKeyRequest::new);
+    static {
+        PARSER.declareString((req, str) -> req.getGrant().setType(str), new ParseField("grant_type"));
+        PARSER.declareString((req, str) -> req.getGrant().setUsername(str), new ParseField("username"));
+        PARSER.declareField((req, secStr) -> req.getGrant().setPassword(secStr), RestGrantApiKeyAction::getSecureString,
+            new ParseField("password"), ObjectParser.ValueType.STRING);
+        PARSER.declareField((req, secStr) -> req.getGrant().setAccessToken(secStr), RestGrantApiKeyAction::getSecureString,
+            new ParseField("access_token"), ObjectParser.ValueType.STRING);
+        PARSER.declareObject((req, api) -> req.setApiKeyRequest(api), (parser, ignore) -> CreateApiKeyRequestBuilder.parse(parser),
+            new ParseField("api_key"));
+    }
+
+    private static SecureString getSecureString(XContentParser parser) throws IOException {
+        return new SecureString(
+            Arrays.copyOfRange(parser.textCharacters(), parser.textOffset(), parser.textOffset() + parser.textLength()));
+    }
+
+    public RestGrantApiKeyAction(Settings settings, XPackLicenseState licenseState) {
+        super(settings, licenseState);
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(POST, "/_security/api_key/grant"),
+            new Route(PUT, "/_security/api_key/grant"));
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_security_grant_api_key";
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        String refresh = request.param("refresh");
+        try (XContentParser parser = request.contentParser()) {
+            final GrantApiKeyRequest grantRequest = PARSER.parse(parser, null);
+            if (refresh != null) {
+                grantRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(refresh));
+            }
+            return channel -> client.execute(GrantApiKeyAction.INSTANCE, grantRequest,
+                ActionListener.delegateResponse(new RestToXContentListener<>(channel), (listener, ex) -> {
+                    RestStatus status = ExceptionsHelper.status(ex);
+                    if (status == RestStatus.UNAUTHORIZED) {
+                        listener.onFailure(
+                            new ElasticsearchSecurityException("Failed to authenticate api key grant", RestStatus.FORBIDDEN, ex));
+                    } else {
+                        listener.onFailure(ex);
+                    }
+                }));
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java
new file mode 100644
index 0000000000000..8d7faf15371f0
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/TransportGrantApiKeyActionTests.java
@@ -0,0 +1,230 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.action;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyRequest;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.AuthenticationService;
+import org.elasticsearch.xpack.security.authc.TokenServiceMock;
+import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator;
+import org.elasticsearch.xpack.security.test.SecurityMocks;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.List;
+
+import static org.elasticsearch.test.TestMatchers.throwableWithMessage;
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
+public class TransportGrantApiKeyActionTests extends ESTestCase {
+
+    private TransportGrantApiKeyAction action;
+    private ApiKeyGenerator apiKeyGenerator;
+    private AuthenticationService authenticationService;
+    private TokenServiceMock tokenServiceMock;
+    private ThreadPool threadPool;
+
+    @Before
+    public void setupMocks() throws Exception {
+        apiKeyGenerator = mock(ApiKeyGenerator.class);
+        authenticationService = mock(AuthenticationService.class);
+
+        threadPool = new TestThreadPool("TP-" + getTestName());
+        tokenServiceMock = SecurityMocks.tokenService(true, threadPool);
+        final ThreadContext threadContext = threadPool.getThreadContext();
+
+        action = new TransportGrantApiKeyAction(mock(TransportService.class), mock(ActionFilters.class), threadContext,
+            apiKeyGenerator, authenticationService, tokenServiceMock.tokenService);
+    }
+
+    @After
+    public void cleanup() {
+        threadPool.shutdown();
+    }
+
+    public void testGrantApiKeyWithUsernamePassword() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final SecureString password = new SecureString(randomAlphaOfLengthBetween(8, 24).toCharArray());
+        final Authentication authentication = buildAuthentication(username);
+
+        final GrantApiKeyRequest request = mockRequest();
+        request.getGrant().setType("password");
+        request.getGrant().setUsername(username);
+        request.getGrant().setPassword(password);
+
+        final CreateApiKeyResponse response = mockResponse(request);
+
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, arrayWithSize(4));
+
+            assertThat(args[0], equalTo(GrantApiKeyAction.NAME));
+            assertThat(args[1], sameInstance(request));
+            assertThat(args[2], instanceOf(UsernamePasswordToken.class));
+            UsernamePasswordToken token = (UsernamePasswordToken) args[2];
+            assertThat(token.principal(), equalTo(username));
+            assertThat(token.credentials(), equalTo(password));
+
+            ActionListener<Authentication> listener = (ActionListener<Authentication>) args[args.length - 1];
+            listener.onResponse(authentication);
+
+            return null;
+        }).when(authenticationService)
+            .authenticate(eq(GrantApiKeyAction.NAME), same(request), any(UsernamePasswordToken.class), any(ActionListener.class));
+
+        setupApiKeyGenerator(authentication, request, response);
+
+        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        assertThat(future.actionGet(), sameInstance(response));
+    }
+
+    public void testGrantApiKeyWithInvalidUsernamePassword() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final SecureString password = new SecureString(randomAlphaOfLengthBetween(8, 24).toCharArray());
+        final Authentication authentication = buildAuthentication(username);
+
+        final GrantApiKeyRequest request = mockRequest();
+        request.getGrant().setType("password");
+        request.getGrant().setUsername(username);
+        request.getGrant().setPassword(password);
+
+        final CreateApiKeyResponse response = mockResponse(request);
+
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, arrayWithSize(4));
+
+            assertThat(args[0], equalTo(GrantApiKeyAction.NAME));
+            assertThat(args[1], sameInstance(request));
+            assertThat(args[2], instanceOf(UsernamePasswordToken.class));
+            UsernamePasswordToken token = (UsernamePasswordToken) args[2];
+            assertThat(token.principal(), equalTo(username));
+            assertThat(token.credentials(), equalTo(password));
+
+            ActionListener<Authentication> listener = (ActionListener<Authentication>) args[args.length - 1];
+            listener.onFailure(new ElasticsearchSecurityException("authentication failed for testing"));
+
+            return null;
+        }).when(authenticationService)
+            .authenticate(eq(GrantApiKeyAction.NAME), same(request), any(UsernamePasswordToken.class), any(ActionListener.class));
+
+        setupApiKeyGenerator(authentication, request, response);
+
+        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        final ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, future::actionGet);
+        assertThat(exception, throwableWithMessage("authentication failed for testing"));
+
+        verifyZeroInteractions(apiKeyGenerator);
+    }
+
+    public void testGrantApiKeyWithAccessToken() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final TokenServiceMock.MockToken token = tokenServiceMock.mockAccessToken();
+        final Authentication authentication = buildAuthentication(username);
+
+        final GrantApiKeyRequest request = mockRequest();
+        request.getGrant().setType("access_token");
+        request.getGrant().setAccessToken(token.encodedToken);
+
+        final CreateApiKeyResponse response = mockResponse(request);
+
+        tokenServiceMock.defineToken(token, authentication);
+        setupApiKeyGenerator(authentication, request, response);
+
+        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        assertThat(future.actionGet(), sameInstance(response));
+        verifyZeroInteractions(authenticationService);
+    }
+
+    public void testGrantApiKeyWithInvalidatedAccessToken() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final TokenServiceMock.MockToken token = tokenServiceMock.mockAccessToken();
+        final Authentication authentication = buildAuthentication(username);
+
+        final GrantApiKeyRequest request = mockRequest();
+        request.getGrant().setType("access_token");
+        request.getGrant().setAccessToken(token.encodedToken);
+
+        final CreateApiKeyResponse response = mockResponse(request);
+
+        tokenServiceMock.defineToken(token, authentication, false);
+        setupApiKeyGenerator(authentication, request, response);
+
+        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        final ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, future::actionGet);
+        assertThat(exception, throwableWithMessage("token expired"));
+
+        verifyZeroInteractions(authenticationService);
+        verifyZeroInteractions(apiKeyGenerator);
+    }
+
+    private Authentication buildAuthentication(String username) {
+        return new Authentication(new User(username),
+            new Authentication.RealmRef("realm_name", "realm_type", "node_name"), null);
+    }
+
+    private CreateApiKeyResponse mockResponse(GrantApiKeyRequest request) {
+        return new CreateApiKeyResponse(request.getApiKeyRequest().getName(),
+            randomAlphaOfLength(12), new SecureString(randomAlphaOfLength(18).toCharArray()), null);
+    }
+
+    private GrantApiKeyRequest mockRequest() {
+        final String keyName = randomAlphaOfLengthBetween(6, 32);
+        final GrantApiKeyRequest request = new GrantApiKeyRequest();
+        request.setApiKeyRequest(new CreateApiKeyRequest(keyName, List.of(), null));
+        return request;
+    }
+
+    private void setupApiKeyGenerator(Authentication authentication, GrantApiKeyRequest request, CreateApiKeyResponse response) {
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, arrayWithSize(3));
+
+            assertThat(args[0], equalTo(authentication));
+            assertThat(args[1], sameInstance(request.getApiKeyRequest()));
+
+            ActionListener<CreateApiKeyResponse> listener = (ActionListener<CreateApiKeyResponse>) args[args.length - 1];
+            listener.onResponse(response);
+
+            return null;
+        }).when(apiKeyGenerator).generateApiKey(any(Authentication.class), any(CreateApiKeyRequest.class), any(ActionListener.class));
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
new file mode 100644
index 0000000000000..76e321f69c439
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
+import org.elasticsearch.xpack.security.test.SecurityMocks;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Because {@link TokenService} is {@code final}, we can't mock it.
+ * Instead, we use this class to control the client that underlies the token service and trigger certain conditions
+ */
+public class TokenServiceMock {
+    public final TokenService tokenService;
+    public final Client client;
+
+    public final class MockToken {
+        public final String baseToken;
+        public final SecureString encodedToken;
+        public final String hashedToken;
+
+        public MockToken(String baseToken, SecureString encodedToken, String hashedToken) {
+            this.baseToken = baseToken;
+            this.encodedToken = encodedToken;
+            this.hashedToken = hashedToken;
+        }
+    }
+
+    public TokenServiceMock(TokenService tokenService, Client client) {
+        this.tokenService = tokenService;
+        this.client = client;
+    }
+
+    public MockToken mockAccessToken() throws Exception {
+        final String uuid = UUIDs.randomBase64UUID();
+        final SecureString encoded = new SecureString(tokenService.prependVersionAndEncodeAccessToken(Version.CURRENT, uuid).toCharArray());
+        final String hashedToken = TokenService.hashTokenString(uuid);
+        return new MockToken(uuid, encoded, hashedToken);
+    }
+
+    public void defineToken(MockToken token, Authentication authentication) throws IOException {
+        defineToken(token, authentication, true);
+    }
+
+    public void defineToken(MockToken token, Authentication authentication, boolean valid) throws IOException {
+        Instant expiration = Instant.now().plusSeconds(TimeUnit.MINUTES.toSeconds(20));
+        final UserToken userToken = new UserToken(token.hashedToken, Version.CURRENT, authentication, expiration, Map.of());
+        final Map<String, Object> document = new HashMap<>();
+        document.put("access_token", Map.of("user_token", userToken, "invalidated", valid == false));
+
+        SecurityMocks.mockGetRequest(client, RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, "token_" + token.hashedToken,
+            XContentTestUtils.convertToXContent(document, XContentType.JSON));
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java
new file mode 100644
index 0000000000000..6d9a92d3a625d
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGeneratorTests.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anySetOf;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+public class ApiKeyGeneratorTests extends ESTestCase {
+
+    public void testGenerateApiKeySuccessfully() {
+        final ApiKeyService apiKeyService = mock(ApiKeyService.class);
+        final CompositeRolesStore rolesStore = mock(CompositeRolesStore.class);
+        final ApiKeyGenerator generator = new ApiKeyGenerator(apiKeyService, rolesStore, NamedXContentRegistry.EMPTY);
+        final Set<String> userRoleNames = Sets.newHashSet(randomArray(1, 4, String[]::new, () -> randomAlphaOfLengthBetween(3, 12)));
+        final Authentication authentication = new Authentication(
+            new User("test", userRoleNames.toArray(String[]::new)),
+            new Authentication.RealmRef("realm-name", "realm-type", "node-name"),
+            null);
+        final CreateApiKeyRequest request = new CreateApiKeyRequest("name", null, null);
+
+        final Set<RoleDescriptor> roleDescriptors = randomSubsetOf(userRoleNames).stream()
+            .map(name -> new RoleDescriptor(name, generateRandomStringArray(3, 6, false), null, null))
+            .collect(Collectors.toUnmodifiableSet());
+
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, arrayWithSize(2));
+
+            Set<String> roleNames = (Set<String>) args[0];
+            assertThat(roleNames, equalTo(userRoleNames));
+
+            ActionListener<Set<RoleDescriptor>> listener = (ActionListener<Set<RoleDescriptor>>) args[args.length - 1];
+            listener.onResponse(roleDescriptors);
+            return null;
+        }).when(rolesStore).getRoleDescriptors(anySetOf(String.class), any(ActionListener.class));
+
+        CreateApiKeyResponse response = new CreateApiKeyResponse(
+            "name", randomAlphaOfLength(18), new SecureString(randomAlphaOfLength(24).toCharArray()), null);
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, arrayWithSize(4));
+
+            assertThat(args[0], sameInstance(authentication));
+            assertThat(args[1], sameInstance(request));
+            assertThat(args[2], sameInstance(roleDescriptors));
+
+            ActionListener<CreateApiKeyResponse> listener = (ActionListener<CreateApiKeyResponse>) args[args.length - 1];
+            listener.onResponse(response);
+
+            return null;
+        }).when(apiKeyService).createApiKey(same(authentication), same(request), anySetOf(RoleDescriptor.class), any(ActionListener.class));
+
+        final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
+        generator.generateApiKey(authentication, request, future);
+
+        assertThat(future.actionGet(), sameInstance(response));
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index aa3b612e513ba..c766ab211c173 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.security.authz;
 
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -15,6 +16,7 @@
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.transport.TransportRequest;
@@ -127,7 +129,7 @@ public void testValidateSearchContext() throws Exception {
             final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
             SearchContextMissingException expected =
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
-            assertEquals(testSearchContext.id(), expected.id());
+            assertEquals(testSearchContext.id(), expected.contextId());
             verify(licenseState, times(3)).isAuthAllowed();
             verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                 authzInfoRoles(authentication.getUser().roles()));
@@ -163,7 +165,7 @@ public void testValidateSearchContext() throws Exception {
             final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
             SearchContextMissingException expected =
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
-            assertEquals(testSearchContext.id(), expected.id());
+            assertEquals(testSearchContext.id(), expected.contextId());
             verify(licenseState, times(5)).isAuthAllowed();
             verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                 authzInfoRoles(authentication.getUser().roles()));
@@ -174,13 +176,13 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication original = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
         Authentication current =
                 randomBoolean() ? original : new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
-        long id = randomLong();
+        SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomLong());
         final String action = randomAlphaOfLength(4);
         TransportRequest request = Empty.INSTANCE;
         AuditTrailService auditTrail = mock(AuditTrailService.class);
 
         final String auditId = randomAlphaOfLengthBetween(8, 20);
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId,
+        ensureAuthenticatedUserIsSame(original, current, auditTrail, contextId, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -188,7 +190,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         User user = new User(new User("test", "role"), new User("authenticated", "runas"));
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId,
+        ensureAuthenticatedUserIsSame(original, current, auditTrail, contextId, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -196,7 +198,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
         Authentication runAs = current;
-        ensureAuthenticatedUserIsSame(runAs, current, auditTrail, id, action, request, auditId,
+        ensureAuthenticatedUserIsSame(runAs, current, auditTrail, contextId, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -204,9 +206,9 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication differentRealmType =
                 new Authentication(new User("test", "role"), new RealmRef("realm", randomAlphaOfLength(5), "node"), null);
         SearchContextMissingException e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, differentRealmType, auditTrail, id, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, differentRealmType, auditTrail, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(id, e.id());
+        assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(differentRealmType), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
 
@@ -214,9 +216,9 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication differentUser =
                 new Authentication(new User("test2", "role"), new RealmRef("realm", "realm", "node"), null);
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, differentUser, auditTrail, id, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, differentUser, auditTrail, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(id, e.id());
+        assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(differentUser), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
 
@@ -224,18 +226,18 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")),
                 new RealmRef("realm", "file", "node1"), new RealmRef("realm", "file", "node1"));
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, diffRunAs, auditTrail, id, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, diffRunAs, auditTrail, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(id, e.id());
+        assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(diffRunAs), eq(action), eq(request), authzInfoRoles(original.getUser().roles()));
 
         // run as different looked up by type
         Authentication runAsDiffType = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node"));
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(runAs, runAsDiffType, auditTrail, id, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(runAs, runAsDiffType, auditTrail, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
-        assertEquals(id, e.id());
+        assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(runAsDiffType), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java
index 0270a07d9f97c..c9bb7879147b6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java
@@ -17,17 +17,29 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.security.authc.TokenService;
+import org.elasticsearch.xpack.security.authc.TokenServiceMock;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.Assert;
 
+import java.security.GeneralSecurityException;
+import java.time.Clock;
+import java.time.Instant;
 import java.util.function.Consumer;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
+import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_TOKENS_ALIAS;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -129,4 +141,20 @@ public static void mockIndexRequest(Client client, String indexAliasName, Consum
             return null;
         }).when(client).execute(eq(IndexAction.INSTANCE), any(IndexRequest.class), any(ActionListener.class));
     }
+
+    public static TokenServiceMock tokenService(boolean enabled, ThreadPool threadPool) throws GeneralSecurityException {
+        final Settings settings = Settings.builder().put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), enabled).build();
+        final Instant now = Instant.now();
+        final Clock clock = Clock.fixed(now, ESTestCase.randomZone());
+        final Client client = mock(Client.class);
+        when(client.threadPool()).thenReturn(threadPool);
+        final XPackLicenseState licenseState = mock(XPackLicenseState.class);
+        when(licenseState.isTokenServiceAllowed()).thenReturn(true);
+        final ClusterService clusterService = mock(ClusterService.class);
+
+        final SecurityContext securityContext = new SecurityContext(settings, threadPool.getThreadContext());
+        final TokenService service = new TokenService(settings, clock, client, licenseState, securityContext,
+            mockSecurityIndexManager(SECURITY_MAIN_ALIAS), mockSecurityIndexManager(SECURITY_TOKENS_ALIAS), clusterService);
+        return new TokenServiceMock(service, client);
+    }
 }
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java
index 6c5725c0b7528..365268396c10c 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java
@@ -5,9 +5,10 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ConnectionConfiguration;
-import org.elasticsearch.xpack.sql.client.Version;
 
+import javax.sql.DataSource;
 import java.io.PrintWriter;
 import java.sql.Connection;
 import java.sql.SQLException;
@@ -16,8 +17,6 @@
 import java.util.Properties;
 import java.util.logging.Logger;
 
-import javax.sql.DataSource;
-
 /**
  * Factory for connections to Elasticsearch SQL.
  */
@@ -25,7 +24,7 @@ public class EsDataSource implements DataSource, Wrapper {
 
     static {
         // invoke Version to perform classpath/jar sanity checks
-        Version.CURRENT.toString();
+        ClientVersion.CURRENT.toString();
     }
 
     private String url;
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java
index 238a8fea163cf..1599fa2ad02af 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
-import org.elasticsearch.xpack.sql.client.Version;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 
 import java.io.PrintWriter;
 import java.sql.Connection;
@@ -23,7 +23,7 @@ public class EsDriver implements Driver {
 
     static {
         // invoke Version to perform classpath/jar sanity checks
-        Version.CURRENT.toString();
+        ClientVersion.CURRENT.toString();
 
         try {
             register();
@@ -96,12 +96,12 @@ public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws
 
     @Override
     public int getMajorVersion() {
-        return Version.CURRENT.major;
+        return ClientVersion.CURRENT.major;
     }
 
     @Override
     public int getMinorVersion() {
-        return Version.CURRENT.minor;
+        return ClientVersion.CURRENT.minor;
     }
 
     @Override
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
index 0b96204fd5d7e..9eae74d18019a 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
@@ -25,6 +25,7 @@ public enum EsType implements SQLType {
     SCALED_FLOAT(             Types.FLOAT),
     KEYWORD(                  Types.VARCHAR),
     TEXT(                     Types.VARCHAR),
+    CONSTANT_KEYWORD(         Types.VARCHAR),
     OBJECT(                   Types.STRUCT),
     NESTED(                   Types.STRUCT),
     BINARY(                   Types.VARBINARY),
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java
index aff896d1c21f3..4ca094b6588df 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/InfoResponse.java
@@ -5,28 +5,23 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
+
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
+
 /**
  * General information about the server.
  */
 class InfoResponse {
     final String cluster;
-    final int majorVersion;
-    final int minorVersion;
-    final int revisionVersion;
+    final SqlVersion version;
 
-    InfoResponse(String clusterName, byte versionMajor, byte versionMinor, byte revisionVersion) {
+    InfoResponse(String clusterName, SqlVersion version) {
         this.cluster = clusterName;
-        this.majorVersion = versionMajor;
-        this.minorVersion = versionMinor;
-        this.revisionVersion = revisionVersion;
+        this.version = version;
     }
 
     @Override
     public String toString() {
-        return cluster + "[" + versionString() + "]";
-    }
-    
-    public String versionString() {
-        return majorVersion + "." + minorVersion + "." + revisionVersion;
+        return cluster + "[" + version.toString() + "]";
     }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
index c9480dbcb1c2b..8fd4c6119b434 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ConnectionConfiguration;
 import org.elasticsearch.xpack.sql.client.StringUtils;
-import org.elasticsearch.xpack.sql.client.Version;
 
 import java.net.URI;
 import java.sql.DriverPropertyInfo;
@@ -70,7 +70,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
         // typically this should have already happened but in case the
         // EsDriver/EsDataSource are not used and the impl. classes used directly
         // this covers that case
-        Version.CURRENT.toString();
+        ClientVersion.CURRENT.toString();
     }
 
     // immutable properties
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java
index 09096fbe405a0..e4ab36996804f 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConnection.java
@@ -426,10 +426,10 @@ String getUserName() {
     // in fact, this information is cached by the underlying client
     // once retrieved
     int esInfoMajorVersion() throws SQLException {
-        return client.serverInfo().majorVersion;
+        return client.serverInfo().version.major;
     }
 
     int esInfoMinorVersion() throws SQLException {
-        return client.serverInfo().minorVersion;
+        return client.serverInfo().version.minor;
     }
 }
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java
index 8fd4417f4a9b0..77563b60041db 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ObjectUtils;
-import org.elasticsearch.xpack.sql.client.Version;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -94,7 +94,7 @@ public String getDatabaseProductName() throws SQLException {
 
     @Override
     public String getDatabaseProductVersion() throws SQLException {
-        return Version.CURRENT.toString();
+        return ClientVersion.CURRENT.toString();
     }
 
     @Override
@@ -104,17 +104,17 @@ public String getDriverName() throws SQLException {
 
     @Override
     public String getDriverVersion() throws SQLException {
-        return Version.CURRENT.major + "." + Version.CURRENT.minor;
+        return ClientVersion.CURRENT.major + "." + ClientVersion.CURRENT.minor;
     }
 
     @Override
     public int getDriverMajorVersion() {
-        return Version.CURRENT.major;
+        return ClientVersion.CURRENT.major;
     }
 
     @Override
     public int getDriverMinorVersion() {
-        return Version.CURRENT.minor;
+        return ClientVersion.CURRENT.minor;
     }
 
     @Override
@@ -1111,12 +1111,12 @@ public int getDatabaseMinorVersion() throws SQLException {
 
     @Override
     public int getJDBCMajorVersion() throws SQLException {
-        return Version.jdbcMajorVersion();
+        return ClientVersion.jdbcMajorVersion();
     }
 
     @Override
     public int getJDBCMinorVersion() throws SQLException {
-        return Version.jdbcMinorVersion();
+        return ClientVersion.jdbcMinorVersion();
     }
 
     @Override
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
index 925187db405aa..684f32981dabc 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
@@ -7,8 +7,8 @@
 
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.HttpClient;
-import org.elasticsearch.xpack.sql.client.Version;
 import org.elasticsearch.xpack.sql.proto.ColumnInfo;
 import org.elasticsearch.xpack.sql.proto.MainResponse;
 import org.elasticsearch.xpack.sql.proto.Mode;
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.sql.proto.SqlQueryRequest;
 import org.elasticsearch.xpack.sql.proto.SqlQueryResponse;
 import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -62,7 +63,7 @@ Cursor query(String sql, List<SqlTypedParamValue> params, RequestMeta meta) thro
                 null,
                 Boolean.FALSE,
                 null,
-                new RequestInfo(Mode.JDBC),
+                new RequestInfo(Mode.JDBC, ClientVersion.CURRENT),
                 conCfg.fieldMultiValueLeniency(),
                 conCfg.indexIncludeFrozen(),
                 conCfg.binaryCommunication());
@@ -94,16 +95,15 @@ InfoResponse serverInfo() throws SQLException {
 
     private InfoResponse fetchServerInfo() throws SQLException {
         MainResponse mainResponse = httpClient.serverInfo();
-        Version version = Version.fromString(mainResponse.getVersion());
-        return new InfoResponse(mainResponse.getClusterName(), version.major, version.minor, version.revision);
+        SqlVersion version = SqlVersion.fromString(mainResponse.getVersion());
+        return new InfoResponse(mainResponse.getClusterName(), version);
     }
-    
+
     private void checkServerVersion() throws SQLException {
-        if (serverInfo.majorVersion != Version.CURRENT.major
-                || serverInfo.minorVersion != Version.CURRENT.minor
-                || serverInfo.revisionVersion != Version.CURRENT.revision) {
+        if (ClientVersion.isServerCompatible(serverInfo.version) == false) {
             throw new SQLException("This version of the JDBC driver is only compatible with Elasticsearch version " +
-                    Version.CURRENT.toString() + ", attempting to connect to a server version " + serverInfo.versionString());
+                ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server version " +
+                serverInfo.version.toString());
         }
     }
 
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
index bf72b4454e40e..888fb8632d217 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
@@ -210,6 +210,7 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
             case BOOLEAN:
             case TEXT:
             case KEYWORD:
+            case CONSTANT_KEYWORD:
                 return v; // These types are already represented correctly in JSON
             case BYTE:
                 return ((Number) v).byteValue(); // Parser might return it as integer or long - need to update to the correct type
@@ -323,6 +324,7 @@ private static Boolean asBoolean(Object val, EsType columnType, String typeStrin
                 return Boolean.valueOf(Integer.signum(((Number) val).intValue()) != 0);
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 return Boolean.valueOf((String) val);
             default:
                 return failConversion(val, columnType, typeString, Boolean.class);
@@ -345,6 +347,7 @@ private static Byte asByte(Object val, EsType columnType, String typeString) thr
                 return safeToByte(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Byte.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -372,6 +375,7 @@ private static Short asShort(Object val, EsType columnType, String typeString) t
                 return safeToShort(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Short.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -398,6 +402,7 @@ private static Integer asInteger(Object val, EsType columnType, String typeStrin
                 return safeToInt(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Integer.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -429,6 +434,7 @@ private static Long asLong(Object val, EsType columnType, String typeString) thr
             //    return ((Number) val).longValue();
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Long.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -456,6 +462,7 @@ private static Float asFloat(Object val, EsType columnType, String typeString) t
                 return Float.valueOf(((Number) val).floatValue());
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Float.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -482,6 +489,7 @@ private static Double asDouble(Object val, EsType columnType, String typeString)
                 return Double.valueOf(((Number) val).doubleValue());
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Double.valueOf((String) val);
                 } catch (NumberFormatException e) {
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
index 511cf92be5665..0fc929c900bfe 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
@@ -76,6 +76,7 @@ private TypeUtils() {}
         types.put(EsType.SCALED_FLOAT, Double.class);
         types.put(EsType.KEYWORD, String.class);
         types.put(EsType.TEXT, String.class);
+        types.put(EsType.CONSTANT_KEYWORD, String.class);
         types.put(EsType.BINARY, byte[].class);
         types.put(EsType.DATETIME, Timestamp.class);
         types.put(EsType.IP, String.class);
@@ -157,7 +158,7 @@ static EsType of(String name) throws SQLException {
     }
 
     static boolean isString(EsType dataType) {
-        return dataType == EsType.KEYWORD || dataType == EsType.TEXT;
+        return dataType == EsType.KEYWORD || dataType == EsType.TEXT || dataType == EsType.CONSTANT_KEYWORD;
     }
 
     static EsType of(Class<? extends Object> clazz) throws SQLException {
diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java
index 16df417855d12..4f6cdd6b51225 100644
--- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java
+++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 
 import java.io.IOException;
 import java.sql.SQLException;
@@ -31,9 +32,9 @@ public void testExceptionThrownOnIncompatibleVersions() throws IOException, SQLE
         
         String url = JdbcConfiguration.URL_PREFIX + webServerAddress();
         SQLException ex = expectThrows(SQLException.class, () -> new JdbcHttpClient(JdbcConfiguration.create(url, null, 0)));
-        assertEquals("This version of the JDBC driver is only compatible with Elasticsearch version "
-                + org.elasticsearch.xpack.sql.client.Version.CURRENT.toString()
-                + ", attempting to connect to a server version " + version.toString(), ex.getMessage());
+        assertEquals("This version of the JDBC driver is only compatible with Elasticsearch version " +
+            ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server " +
+            "version " + version.toString(), ex.getMessage());
     }
     
     public void testNoExceptionThrownForCompatibleVersions() throws IOException {
diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java
index 0a2d3206468d8..8a47025a17135 100644
--- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java
+++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java
@@ -6,15 +6,14 @@
 package org.elasticsearch.xpack.sql.jdbc;
 
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.sql.client.Version;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 
 public class VersionTests extends ESTestCase {
     public void testVersionIsCurrent() {
         /* This test will only work properly in gradle because in gradle we run the tests
          * using the jar. */
-        assertNotNull(Version.CURRENT.hash);
-        assertEquals(org.elasticsearch.Version.CURRENT.major, Version.CURRENT.major);
-        assertEquals(org.elasticsearch.Version.CURRENT.minor, Version.CURRENT.minor);
-        assertEquals(org.elasticsearch.Version.CURRENT.revision, Version.CURRENT.revision);
+        assertEquals(org.elasticsearch.Version.CURRENT.major, ClientVersion.CURRENT.major);
+        assertEquals(org.elasticsearch.Version.CURRENT.minor, ClientVersion.CURRENT.minor);
+        assertEquals(org.elasticsearch.Version.CURRENT.revision, ClientVersion.CURRENT.revision);
     }
 }
diff --git a/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
index 01ef407d4a027..91f1a044c59f5 100644
--- a/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
@@ -26,6 +26,7 @@
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.toMap;
+import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
 
@@ -110,7 +111,7 @@ private void assertCount(RestClient client, int count) throws IOException {
         expected.put("rows", singletonList(singletonList(count)));
 
         Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
-        request.setJsonEntity("{\"query\": \"SELECT COUNT(*) FROM test\"" + mode(mode) + "}");
+        request.setJsonEntity("{\"query\": \"SELECT COUNT(*) FROM test\"" + mode(mode) + version(mode) + "}");
         Map<String, Object> actual = toMap(client.performRequest(request), mode);
 
         if (false == expected.equals(actual)) {
diff --git a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
index f29f8ae423cf3..28c3b8fc018ac 100644
--- a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
@@ -32,6 +32,7 @@
 
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
+import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
 import static org.hamcrest.Matchers.containsString;
@@ -70,10 +71,10 @@ public void expectMatchesAdmin(String adminSql, String user, String userSql) thr
         public void expectScrollMatchesAdmin(String adminSql, String user, String userSql) throws Exception {
             String mode = randomMode();
             Map<String, Object> adminResponse = runSql(null,
-                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + "}",
+                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + version(mode) + "}",
                             ContentType.APPLICATION_JSON), mode);
             Map<String, Object> otherResponse = runSql(user,
-                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + "}",
+                    new StringEntity("{\"query\": \"" + adminSql + "\", \"fetch_size\": 1" + mode(mode) + version(mode) + "}",
                             ContentType.APPLICATION_JSON), mode);
 
             String adminCursor = (String) adminResponse.remove("cursor");
@@ -83,9 +84,11 @@ public void expectScrollMatchesAdmin(String adminSql, String user, String userSq
             assertResponse(adminResponse, otherResponse);
             while (true) {
                 adminResponse = runSql(null,
-                        new StringEntity("{\"cursor\": \"" + adminCursor + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), mode);
+                        new StringEntity("{\"cursor\": \"" + adminCursor + "\"" + mode(mode) + version(mode) + "}",
+                            ContentType.APPLICATION_JSON), mode);
                 otherResponse = runSql(user,
-                        new StringEntity("{\"cursor\": \"" + otherCursor + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), mode);
+                        new StringEntity("{\"cursor\": \"" + otherCursor + "\"" + mode(mode) + version(mode) + "}",
+                            ContentType.APPLICATION_JSON), mode);
                 adminCursor = (String) adminResponse.remove("cursor");
                 otherCursor = (String) otherResponse.remove("cursor");
                 assertResponse(adminResponse, otherResponse);
@@ -180,7 +183,8 @@ public void checkNoMonitorMain(String user) throws Exception {
         }
 
         private static Map<String, Object> runSql(@Nullable String asUser, String mode, String sql) throws IOException {
-            return runSql(asUser, new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), mode);
+            return runSql(asUser, new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + version(mode) + "}",
+                ContentType.APPLICATION_JSON), mode);
         }
 
         private static Map<String, Object> runSql(@Nullable String asUser, HttpEntity entity, String mode) throws IOException {
@@ -230,18 +234,18 @@ protected AuditLogAsserter createAuditLogAsserter() {
      */
     public void testHijackScrollFails() throws Exception {
         createUser("full_access", "rest_minimal");
+        final String mode = randomMode();
 
-        String mode = randomMode();
         Map<String, Object> adminResponse = RestActions.runSql(null,
-                new StringEntity("{\"query\": \"SELECT * FROM test\", \"fetch_size\": 1" + mode(mode) + "}",
+                new StringEntity("{\"query\": \"SELECT * FROM test\", \"fetch_size\": 1" + mode(mode) + version(mode) + "}",
                         ContentType.APPLICATION_JSON), mode);
 
         String cursor = (String) adminResponse.remove("cursor");
         assertNotNull(cursor);
 
-        final String m = randomMode();
         ResponseException e = expectThrows(ResponseException.class, () -> RestActions.runSql("full_access",
-                new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(m) + "}", ContentType.APPLICATION_JSON), m));
+                new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) + "}", ContentType.APPLICATION_JSON),
+                mode));
         // TODO return a better error message for bad scrolls
         assertThat(e.getMessage(), containsString("No search context found for id"));
         assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
diff --git a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
index e2f14ffed6c40..96dfcec9150c0 100644
--- a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java
@@ -33,6 +33,7 @@
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.mode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.randomMode;
 import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.toMap;
+import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
 
@@ -177,7 +178,8 @@ private Map<String, Object> runSql(String asUser, String mode, String sql) throw
             options.addHeader("es-security-runas-user", asUser);
             request.setOptions(options);
         }
-        request.setEntity(new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON));
+        request.setEntity(new StringEntity("{\"query\": \"" + sql + "\"" + mode(mode) + version(mode) + "}",
+            ContentType.APPLICATION_JSON));
         return toMap(client().performRequest(request), mode);
     }
     
diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
index 3765c45562153..cf8849bcab573 100644
--- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
@@ -42,16 +42,16 @@ public void testErrorMessageForTranslatingSQLCommandStatement() throws IOExcepti
 
     public void testErrorMessageForInvalidParamDataType() throws IOException {
         expectBadRequest(() -> runTranslateSql(
-            "{\"query\":\"SELECT null WHERE 0 = ? \", \"mode\": \"odbc\", \"params\":[{\"type\":\"invalid\", \"value\":\"irrelevant\"}]}"
-            ),
+            "{\"query\":\"SELECT null WHERE 0 = ? \"" + mode("odbc") + version("odbc") +
+                ", \"params\":[{\"type\":\"invalid\", \"value\":\"irrelevant\"}]}"),
             containsString("Invalid parameter data type [invalid]")
         );
     }
 
     public void testErrorMessageForInvalidParamSpec() throws IOException {
         expectBadRequest(() -> runTranslateSql(
-            "{\"query\":\"SELECT null WHERE 0 = ? \", \"mode\": \"odbc\", \"params\":[{\"type\":\"SHAPE\", \"value\":false}]}"
-            ),
+            "{\"query\":\"SELECT null WHERE 0 = ? \"" + mode("odbc") + version("odbc") +
+                ", \"params\":[{\"type\":\"SHAPE\", \"value\":false}]}"),
             containsString("Cannot cast value [false] of type [BOOLEAN] to parameter type [SHAPE]")
         );
     }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
index 4a760b97cd23c..150db29a2f58e 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
@@ -103,6 +103,40 @@ public void testKeywordField() throws IOException {
         assertResponse(expected, runSql("SELECT keyword_field FROM test"));
     }
 
+    /*
+     *    "constant_keyword_field": {
+     *       "type": "constant_keyword",
+     *       "value": "foo"
+     *    }
+     */
+    public void testConstantKeywordField() throws IOException {
+        String value = randomAlphaOfLength(20);
+        // _source for `constant_keyword` fields doesn't matter, as they should be taken from docvalue_fields
+        boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting
+        boolean enableSource = randomBoolean();          // enable _source at index level
+        
+        Map<String, Object> indexProps = new HashMap<>(1);
+        indexProps.put("_source", enableSource);
+        
+        Map<String, Map<String, Object>> fieldProps = null;
+        if (randomBoolean()) {
+            fieldProps = new HashMap<>(1);
+            Map<String, Object> fieldProp = new HashMap<>(1);
+            fieldProp.put("value", value);
+            fieldProps.put("constant_keyword_field", fieldProp);
+        }
+        
+        createIndexWithFieldTypeAndProperties("constant_keyword", fieldProps, explicitSourceSetting ? indexProps : null);
+        index("{\"constant_keyword_field\":\"" + value + "\"}");
+        
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("columns", Arrays.asList(
+                columnInfo("plain", "constant_keyword_field", "constant_keyword", JDBCType.VARCHAR, Integer.MAX_VALUE)
+        ));
+        expected.put("rows", singletonList(singletonList(value)));
+        assertResponse(expected, runSql("SELECT constant_keyword_field FROM test"));
+    }
+
     /*
      *    "long/integer/short/byte_field": {
      *       "type": "long/integer/short/byte"
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java
index d35f93f471884..4a3b779cae0f5 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java
@@ -31,6 +31,7 @@
 import static org.elasticsearch.xpack.sql.proto.Mode.CLI;
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS;
+import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
 
 public abstract class SqlProtocolTestCase extends ESRestTestCase {
@@ -135,7 +136,7 @@ private void assertFloatingPointNumbersReturnTypes(Request request, Mode mode) t
                 + "CAST(-1234.34 AS REAL) AS float_negative,"
                 + "1234567890123.34 AS double_positive,"
                 + "-1234567890123.34 AS double_negative\""
-                + mode(mode.toString()) + "}";
+                + mode(mode.toString()) + version(mode.toString()) + "}";
         request.setEntity(new StringEntity(requestContent, ContentType.APPLICATION_JSON));
         
         Map<String, Object> map;
@@ -188,7 +189,7 @@ private void assertQuery(String sql, String columnName, String columnType, Objec
     private void assertQuery(String sql, String columnName, String columnType, Object columnValue, int displaySize, Mode mode)
             throws IOException {
         boolean columnar = randomBoolean();
-        Map<String, Object> response = runSql(mode.toString(), sql, columnar);
+        Map<String, Object> response = runSql(mode, sql, columnar);
         List<Object> columns = (ArrayList<Object>) response.get("columns");
         assertEquals(1, columns.size());
 
@@ -215,12 +216,12 @@ private void assertQuery(String sql, String columnName, String columnType, Objec
             assertEquals(columnValue, row.get(0));
         }
     }
-    
-    private Map<String, Object> runSql(String mode, String sql, boolean columnar) throws IOException {
+
+    private Map<String, Object> runSql(Mode mode, String sql, boolean columnar) throws IOException {
         Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
-        String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode) + "}";
+        String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode.toString()) + version(mode.toString()) + "}";
         String format = randomFrom(XContentType.values()).name().toLowerCase(Locale.ROOT);
-        
+
         // add a client_id to the request
         if (randomBoolean()) {
             String clientId = randomFrom(randomFrom(CLIENT_IDS), randomAlphaOfLengthBetween(10, 20));
@@ -252,14 +253,13 @@ private Map<String, Object> runSql(String mode, String sql, boolean columnar) th
 
         // randomize binary response enforcement for drivers (ODBC/JDBC) and CLI
         boolean binaryCommunication = randomBoolean();
-        Mode m = Mode.fromString(mode);
         if (randomBoolean()) {
             // set it explicitly or leave the default (null) as is
             requestContent = new StringBuilder(requestContent)
                     .insert(requestContent.length() - 1, ",\"binary_format\":" + binaryCommunication).toString();
-            binaryCommunication = ((Mode.isDriver(m) || m == Mode.CLI) && binaryCommunication);
+            binaryCommunication = Mode.isDedicatedClient(mode) && binaryCommunication;
         } else {
-            binaryCommunication = Mode.isDriver(m) || m == Mode.CLI;
+            binaryCommunication = Mode.isDedicatedClient(mode);
         }
         
         // send the query either as body or as request parameter
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
index d2dffe21e1013..e362349f332d3 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
@@ -123,13 +123,11 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                 createString("first_name", createIndex);
                 createString("last_name", createIndex);
                 createIndex.startObject("gender").field("type", "keyword");
-                if (extraFields) {
-                    createIndex.field("copy_to", "extra_gender");
-                }
                 createIndex.endObject();
 
                 if (extraFields) {
-                    createIndex.startObject("extra_gender").field("type", "keyword").endObject();
+                    createIndex.startObject("extra_gender").field("type", "constant_keyword").endObject();
+                    createIndex.startObject("null_constant").field("type", "constant_keyword").endObject();
                     createIndex.startObject("extra.info.gender")
                         .field("type", "alias")
                         .field("path", "gender")
@@ -198,6 +196,9 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                     }
                     hadLastItem = true;
                     bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"');
+                    if (titles.get(f).equals("gender") && extraFields) {
+                        bulk.append(",\"extra_gender\":\"Female\"");
+                    }
                 }
             }
             // append department
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java
index 9c30d80249f75..b2260ba29085b 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.sql.qa.rest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -38,6 +39,14 @@ public static String mode(String mode) {
         return Strings.isEmpty(mode) ? StringUtils.EMPTY : ",\"mode\":\"" + mode + "\"";
     }
 
+    public static String version(String mode) {
+        Mode m = Mode.fromString(mode);
+        if (Mode.isDedicatedClient(m)) {
+            return ",\"version\":" + "\"" + Version.CURRENT.toString() + "\"";
+        }
+        return StringUtils.EMPTY;
+    }
+
     public static String randomMode() {
         return randomFrom(StringUtils.EMPTY, "jdbc", "plain");
     }
@@ -49,7 +58,7 @@ public static String randomMode() {
     public static Number xContentDependentFloatingNumberValue(String mode, Number value) {
         Mode m = Mode.fromString(mode);
         // for drivers and the CLI return the number as is, while for REST cast it implicitly to Double (the JSON standard).
-        if (Mode.isDriver(m) || m == Mode.CLI) {
+        if (Mode.isDedicatedClient(m)) {
             return value;
         } else {
             return value.doubleValue();
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
index 946a1a9a8cac6..0f656dcfcd6e9 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
@@ -105,8 +105,9 @@ public void testNextPage() throws IOException {
                 + "   SELECT text, number, SQRT(number) AS s, SCORE()"
                 + "     FROM test"
                 + " ORDER BY number, SCORE()\", "
-                + "\"mode\":\"" + mode + "\", "
-            + "\"fetch_size\":2" + columnarParameter(columnar) + "}";
+                + "\"mode\":\"" + mode + "\""
+                + version(mode)
+                + ", \"fetch_size\":2" + columnarParameter(columnar) + "}";
 
         Number value = xContentDependentFloatingNumberValue(mode, 1f);
         String cursor = null;
@@ -116,7 +117,8 @@ public void testNextPage() throws IOException {
                 response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "", mode);
             } else {
                 columnar = randomBoolean();
-                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
+                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) +
+                        columnarParameter(columnar) + "}",
                         ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode);
             }
 
@@ -151,7 +153,8 @@ public void testNextPage() throws IOException {
         } else {
             expected.put("rows", emptyList());
         }
-        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
+        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) +
+                columnarParameter(columnar) + "}",
                 ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
@@ -187,8 +190,9 @@ public void testNextPageWithDatetimeAndTimezoneParam() throws IOException {
         String sqlRequest =
                 "{\"query\":\"SELECT DATE_PART('TZOFFSET', date) AS tz FROM test_date_timezone ORDER BY date\","
                         + "\"time_zone\":\"" + zoneId.getId() + "\", "
-                        + "\"mode\":\"" + mode + "\", "
-                        + "\"fetch_size\":2}";
+                        + "\"mode\":\"" + mode + "\""
+                        + version(mode)
+                        + ",\"fetch_size\":2}";
 
         String cursor = null;
         for (int i = 0; i <= datetimes.length; i += 2) {
@@ -199,7 +203,7 @@ public void testNextPageWithDatetimeAndTimezoneParam() throws IOException {
                 expected.put("columns", singletonList(columnInfo(mode, "tz", "integer", JDBCType.INTEGER, 11)));
                 response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "", mode);
             } else {
-                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
+                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) + "}",
                         ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode);
             }
 
@@ -215,7 +219,7 @@ public void testNextPageWithDatetimeAndTimezoneParam() throws IOException {
         }
         Map<String, Object> expected = new HashMap<>();
         expected.put("rows", emptyList());
-        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
+        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + version(mode) + "}",
                 ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
@@ -423,9 +427,10 @@ public void testUseColumnarForUnsupportedFormats() throws Exception {
     
     public void testUseColumnarForTranslateRequest() throws IOException {
         index("{\"test\":\"test\"}", "{\"test\":\"test\"}");
-        
+
+        String mode = randomMode();
         Request request = new Request("POST", SQL_TRANSLATE_REST_ENDPOINT);
-        request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
+        request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(mode) + version(mode) + "}",
                 ContentType.APPLICATION_JSON));
         expectBadRequest(() -> {
                 client().performRequest(request);
@@ -464,7 +469,7 @@ private Map<String, Object> runSql(String mode, String sql, boolean columnar) th
 
     private Map<String, Object> runSql(String mode, String sql, String suffix, boolean columnar) throws IOException {
         // put an explicit "columnar": false parameter or omit it altogether, it should make no difference
-        return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + columnarParameter(columnar) + "}",
+        return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + version(mode) + columnarParameter(columnar) + "}",
                 ContentType.APPLICATION_JSON), suffix, mode);
     }
     
@@ -567,9 +572,10 @@ private void executeAndAssertPrettyPrinting(String expectedJson, String prettyPa
             options.addHeader("Accept", randomFrom("*/*", "application/json"));
             request.setOptions(options);
         }
-        request.setEntity(new StringEntity("{\"query\":\"SELECT * FROM test\"" + mode("plain") + columnarParameter(columnar) + "}",
-                  ContentType.APPLICATION_JSON));
-        
+        request.setEntity(new StringEntity("{\"query\":\"SELECT * FROM test\"" + mode("plain") + version("plain") +
+                columnarParameter(columnar) + "}",
+                ContentType.APPLICATION_JSON));
+
         Response response = client().performRequest(request);
         try (InputStream content = response.getEntity().getContent()) {
             String actualJson = new BytesArray(content.readAllBytes()).utf8ToString();
@@ -598,7 +604,7 @@ public void testBasicQueryWithFilter() throws IOException {
         expected.put("columns", singletonList(columnInfo(mode, "test", "text", JDBCType.VARCHAR, Integer.MAX_VALUE)));
         expected.put("rows", singletonList(singletonList("foo")));
         assertResponse(expected, runSql(new StringEntity("{\"query\":\"SELECT * FROM test\", " +
-                "\"filter\":{\"match\": {\"test\": \"foo\"}}" + mode(mode) + "}",
+                "\"filter\":{\"match\": {\"test\": \"foo\"}}" + mode(mode) + version(mode) + "}",
                 ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
@@ -623,7 +629,7 @@ public void testBasicQueryWithParameters() throws IOException {
             "10, \"foo\"";
         assertResponse(expected, runSql(new StringEntity("{\"query\":\"SELECT test, ? param FROM test WHERE test = ?\", " +
                 "\"params\":[" + params + "]"
-                + mode(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
+                + mode(mode) + version(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode));
     }
 
     public void testBasicTranslateQueryWithFilter() throws IOException {
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
index 4d864961e8ecd..fe3dd9019a4ae 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
@@ -29,6 +29,7 @@
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_STATS_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_TRANSLATE_REST_ENDPOINT;
+import static org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase.version;
 import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.mode;
 
 public abstract class RestSqlUsageTestCase extends ESRestTestCase {
@@ -255,18 +256,18 @@ private void runTranslate(String sql) throws IOException {
     }
     
     private void runSql(String sql) throws IOException {
-        String mode = Mode.PLAIN.toString();
+        Mode mode = Mode.PLAIN;
         if (clientType.equals(ClientType.JDBC.toString())) {
-            mode = Mode.JDBC.toString();
+            mode = Mode.JDBC;
         } else if (clientType.startsWith(ClientType.ODBC.toString())) {
-            mode = Mode.ODBC.toString();
+            mode = Mode.ODBC;
         } else if (clientType.equals(ClientType.CLI.toString())) {
-            mode = Mode.CLI.toString();
+            mode = Mode.CLI;
         }
 
-        runSql(mode, clientType, sql);
+        runSql(mode.toString(), clientType, sql);
     }
-    
+
     @SuppressWarnings({ "unchecked", "rawtypes" })
     private void assertTranslateQueryMetric(int expected, Map<String, Object> responseAsMap) throws IOException {
         List<Map<String, Map<String, Map>>> nodesListStats = (List) responseAsMap.get("stats");
@@ -278,7 +279,7 @@ private void assertTranslateQueryMetric(int expected, Map<String, Object> respon
         }
         assertEquals(expected, actualMetricValue);
     }
-    
+
     private void runSql(String mode, String restClient, String sql) throws IOException {
         Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
         request.addParameter("error_trace", "true");   // Helps with debugging in case something crazy happens on the server.
@@ -293,9 +294,8 @@ private void runSql(String mode, String restClient, String sql) throws IOExcepti
             options.addHeader("Accept", randomFrom("*/*", "application/json"));
             request.setOptions(options);
         }
-        request.setEntity(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) +
-                (ignoreClientType ? StringUtils.EMPTY : ",\"client_id\":\"" + restClient + "\"") + "}",
-                ContentType.APPLICATION_JSON));
+        request.setEntity(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + version(mode) +
+            (ignoreClientType ? StringUtils.EMPTY : ",\"client_id\":\"" + restClient + "\"") + "}", ContentType.APPLICATION_JSON));
         client().performRequest(request);
     }
     
diff --git a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
index 6c51bbf3dc69b..440434796a163 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
@@ -41,7 +41,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -50,6 +50,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
@@ -69,7 +70,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -78,6 +79,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
index b17ba988472be..c8ea838745c46 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
@@ -276,7 +276,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -285,6 +285,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
@@ -304,7 +305,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -313,6 +314,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
new file mode 100644
index 0000000000000..42257a8890ebd
--- /dev/null
+++ b/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
@@ -0,0 +1,163 @@
+// To mute tests follow example in file: example.csv-spec
+
+//
+// Tests testing constant_keyword (introduced in ES 7.7)
+//
+
+// filtering
+
+filterEquals
+SELECT extra_gender g FROM "test_emp_copy" WHERE g = 'Female' LIMIT 5;
+
+    g     
+---------------
+Female              
+Female              
+Female             
+Female         
+Female
+;
+
+filterNotEquals
+SELECT COUNT(*) AS c FROM "test_emp_copy" WHERE extra_gender <> 'Female';
+
+    c:l     
+---------------
+0
+;
+
+aggWithNullFilter
+SELECT COUNT(*) count FROM test_emp_copy WHERE extra_gender IS NOT NULL;
+
+     count:l     
+---------------
+100
+;
+
+// Awaits for https://github.com/elastic/elasticsearch/issues/53545
+aggWithNullFilter-Ignore
+SELECT COUNT(*) count FROM test_emp_copy WHERE null_constant IS NULL;
+
+     count:l     
+---------------
+100
+;
+
+functionOverAlias
+SELECT BIT_LENGTH(extra_gender) bit FROM test_emp_copy ORDER BY extra_gender LIMIT 1;
+
+      bit      
+---------------
+48
+;
+
+
+singlePercentileWithoutComma
+SELECT extra_gender AS gender, PERCENTILE(emp_no, 97) p1 FROM test_emp_copy GROUP BY extra_gender;
+
+gender:s             | p1:d
+Female               |10097.5        
+;
+
+percentileRank
+SELECT extra_gender AS gender, PERCENTILE_RANK(emp_no, 10025) rank FROM test_emp_copy GROUP BY extra_gender;
+
+gender:s       | rank:d
+---------------+---------------
+Female         |24.5           
+;
+
+aggSumWithAliasWithColumnRepeatedWithOrderDesc
+SELECT extra_gender AS g, extra_gender, SUM(salary) AS s3, SUM(salary), SUM(salary) AS s5 FROM test_emp_copy GROUP BY extra_gender;
+
+       g:s     | extra_gender:s |      s3:i     | SUM(salary):i |      s5:i       
+---------------+----------------+---------------+---------------+---------------
+Female         |Female          |4824855        |4824855        |4824855      
+;
+
+topHitsAsMinAndMaxAndGroupBy
+schema::extra_gender:s|min:s|max:s|first:s|last:s
+SELECT extra_gender, MIN(first_name) as min, MAX(first_name) as max, FIRST(first_name) as first, LAST(first_name) as last FROM test_emp_copy GROUP BY extra_gender ORDER BY extra_gender;
+
+ extra_gender  |      min      |      max      |     first     |     last      
+---------------+---------------+---------------+---------------+---------------
+Female         |Alejandro      |Zvonko         |Alejandro      |Zvonko         
+;
+
+medianAbsoluteDeviationOnTwoFields
+schema::extra_gender:s|avg:l|mad_s:l|mad_l:d
+SELECT extra_gender, FLOOR(AVG(salary)) AS avg, FLOOR(MAD(salary)) AS mad_s, MAD(languages) AS mad_l FROM test_emp_copy GROUP BY extra_gender ORDER BY extra_gender;
+
+ extra_gender  |      avg    |     mad_s   |     mad_l     
+---------------+-------------+-------------+---------------
+Female         |48248        |10096        |1            
+;
+
+caseGroupByAndHaving
+schema::count:l|extra_gender:s|languages:byte
+SELECT count(*) AS count, extra_gender, languages FROM test_emp_copy
+GROUP BY 2, 3 HAVING CASE WHEN count(*) > 10 THEN 'many' ELSE 'a few' END = 'many'
+ORDER BY 1, 3;
+
+     count     | extra_gender  |   languages   
+---------------+---------------+---------------
+15             |Female         |1              
+17             |Female         |3              
+18             |Female         |4              
+19             |Female         |2              
+21             |Female         |5              
+;
+
+whereFieldWithRLikeAndGroupByOrderBy
+SELECT last_name l, extra_gender g, COUNT(*) c, MAX(salary) AS sal FROM test_emp_copy WHERE emp_no < 10050 AND (last_name RLIKE 'B.*' OR extra_gender = 'F') GROUP BY g, l ORDER BY sal;
+
+      l:s      |      g:s      |      c:l      |     sal:i      
+---------------+---------------+---------------+---------------
+Berztiss       |Female         |1              |28336        
+Brender        |Female         |1              |36051        
+Bridgland      |Female         |1              |48942        
+Bouloucos      |Female         |1              |58715        
+Bamford        |Female         |1              |61805        
+;
+
+multipleGroupingsAndOrderingByGroupsWithFunctions_2
+SELECT first_name f, last_name l, LCASE(extra_gender) g, CONCAT(UCASE(first_name), LCASE(last_name)) c FROM test_emp_copy GROUP BY f, LCASE(extra_gender), l, c ORDER BY c DESC, first_name, l ASC, g LIMIT 3;
+
+      f:s      |      l:s      |      g:s      |       c:s       
+---------------+---------------+---------------+---------------
+null           |Swan           |female         |swan           
+null           |Reistad        |female         |reistad        
+null           |Portugali      |female         |portugali      
+;
+
+isNotNullWithCount
+SELECT extra_gender IS NOT NULL AS bool, COUNT(*) FROM test_emp_copy GROUP BY extra_gender;
+
+          bool:b        |   COUNT(*):l    
+------------------------+---------------
+true                    |100            
+;
+
+likeWithCount1
+SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender LIKE 'F%';
+
+   COUNT(*):l    
+---------------
+100            
+;
+
+likeWithCount2
+SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender LIKE '%m%';
+
+   COUNT(*):l    
+---------------
+100            
+;
+
+likeWithCount3
+SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender NOT LIKE '%m%';
+
+   COUNT(*):l    
+---------------
+0         
+;
\ No newline at end of file
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
index 711df01849103..dee74303b9bda 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
@@ -29,19 +29,20 @@ SYS COLUMNS TABLE LIKE 'test_emp';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest  |null           |test_emp            |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |birth_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |emp_no             |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra.info.gender  |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_gender       |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_no           |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |gender             |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |hire_date          |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |languages          |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name          |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name.keyword  |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
+integTest  |null           |test_emp            |null_constant      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |salary             |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsWithCatalogAndLike
@@ -49,19 +50,20 @@ SYS COLUMNS CATALOG 'integTest' TABLE LIKE 'test\_emp\_copy' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s  |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest  |null           |test_emp_copy|birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|birth_date        |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|emp_no            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_no          |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name        |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|gender            |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|hire_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|languages         |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
+integTest  |null           |test_emp_copy|null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|salary            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsOnAliasWithTableLike
@@ -69,55 +71,57 @@ SYS COLUMNS TABLE LIKE 'test\_alias' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest   |null           |test_alias    |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |birth_date        |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |emp_no            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_no          |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name        |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |gender            |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |hire_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |languages         |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
+integTest   |null           |test_alias    |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |salary            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsAllTables
 SYS COLUMNS TABLE LIKE '%';
 
-   TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
----------------+---------------+---------------+------------------+---------------+---------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------
-integTest      |null           |logs           |@timestamp        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_in          |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |2                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_out         |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_ip         |12             |IP             |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_port       |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |dest_ip           |12             |IP             |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |id                |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |status            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |birth_date        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |emp_no            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name        |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name.keyword|12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |gender            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |hire_date         |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |languages         |-6             |BYTE           |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name         |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name.keyword |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |salary            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |birth_date        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |emp_no            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_gender      |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_no          |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name        |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |gender            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |hire_date         |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |12                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |languages         |-6             |BYTE           |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |13                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name         |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |14                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |15                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |salary            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |16                |YES          |null           |null           |null           |null            |NO                |NO
+   TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s  |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
+---------------+---------------+---------------+------------------+---------------+----------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------
+integTest      |null           |logs           |@timestamp        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |bytes_in          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |2                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |bytes_out         |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |client_ip         |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |4                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |client_port       |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |5                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |dest_ip           |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |id                |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |status            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |11                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra_no          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |11                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |12                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |13                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |14                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |15                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |16                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |17                |YES          |null           |null           |null           |null            |NO                |NO
 ;
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
new file mode 100644
index 0000000000000..fe30b9975cab4
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
@@ -0,0 +1 @@
+39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
deleted file mode 100644
index 3fd9e819ce181..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-c4475920b08.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ca406661129d35008411365d2b6e747dc39378af
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
index 27712e93205de..d0260551c1c88 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
@@ -5,9 +5,12 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
@@ -33,6 +36,8 @@
 import java.util.Objects;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 /**
  * Base class for requests that contain sql queries (Query and Translate)
  */
@@ -46,7 +51,8 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme
     @Nullable
     private QueryBuilder filter = null;
     private List<SqlTypedParamValue> params = Collections.emptyList();
-    
+
+    // TODO: define all REST request object field names in a protocol class as unique source
     static final ParseField QUERY = new ParseField("query");
     static final ParseField CURSOR = new ParseField("cursor");
     static final ParseField PARAMS = new ParseField("params");
@@ -57,6 +63,7 @@ public abstract class AbstractSqlQueryRequest extends AbstractSqlRequest impleme
     static final ParseField FILTER = new ParseField("filter");
     static final ParseField MODE = new ParseField("mode");
     static final ParseField CLIENT_ID = new ParseField("client_id");
+    static final ParseField CLIENT_VERSION = new ParseField("version");
 
     public AbstractSqlQueryRequest() {
         super();
@@ -80,7 +87,8 @@ protected static <R extends AbstractSqlQueryRequest> ObjectParser<R, Void> objec
         ObjectParser<R, Void> parser = new ObjectParser<>("sql/query", false, supplier);
         parser.declareString(AbstractSqlQueryRequest::query, QUERY);
         parser.declareString((request, mode) -> request.mode(Mode.fromString(mode)), MODE);
-        parser.declareString((request, clientId) -> request.clientId(clientId), CLIENT_ID);
+        parser.declareString(AbstractSqlRequest::clientId, CLIENT_ID);
+        parser.declareString(AbstractSqlRequest::version, CLIENT_VERSION);
         parser.declareField(AbstractSqlQueryRequest::params, AbstractSqlQueryRequest::parseParams, PARAMS, ValueType.VALUE_ARRAY);
         parser.declareString((request, zoneId) -> request.zoneId(ZoneId.of(zoneId)), TIME_ZONE);
         parser.declareInt(AbstractSqlQueryRequest::fetchSize, FETCH_SIZE);
@@ -205,6 +213,26 @@ protected static void validateParams(List<SqlTypedParamValue> params, Mode mode)
         }
     }
 
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        // the version field is mandatory for drivers and CLI
+        Mode mode = requestInfo().mode();
+        if (mode != null && (Mode.isDedicatedClient(mode))) {
+            if (requestInfo().version() == null) {
+                if (Strings.hasText(query())) {
+                    validationException = addValidationError("[version] is required for the [" + mode.toString() + "] client",
+                        validationException);
+                }
+            } else if (requestInfo().version().equals(Version.CURRENT.toString()) == false) {
+                validationException = addValidationError("The [" + requestInfo().version() + "] version of the [" +
+                        mode.toString() + "] " + "client is not compatible with Elasticsearch version [" + Version.CURRENT + "]",
+                    validationException);
+            }
+        }
+        return validationException;
+    }
+
     /**
      * The client's time zone
      */
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java
index f5965cc0e681f..a9fb2f0509fef 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlRequest.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.xpack.sql.proto.Mode;
 import org.elasticsearch.xpack.sql.proto.RequestInfo;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -39,7 +40,8 @@ protected AbstractSqlRequest(StreamInput in) throws IOException {
         super(in);
         Mode mode = in.readEnum(Mode.class);
         String clientId = in.readOptionalString();
-        requestInfo = new RequestInfo(mode, clientId);
+        String clientVersion = in.readOptionalString();
+        requestInfo = new RequestInfo(mode, clientId, clientVersion);
     }
 
     @Override
@@ -56,8 +58,9 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeEnum(requestInfo.mode());
         out.writeOptionalString(requestInfo.clientId());
+        out.writeOptionalString(requestInfo.version() == null ? null : requestInfo.version().toString());
     }
-    
+
     public RequestInfo requestInfo() {
         return requestInfo;
     }
@@ -86,6 +89,14 @@ public void clientId(String clientId) {
         this.requestInfo.clientId(clientId);
     }
 
+    public void version(String clientVersion) {
+        requestInfo.version(clientVersion);
+    }
+
+    public SqlVersion version() {
+        return requestInfo.version();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java
index 563b1fe7cf886..5b2974d84fb4d 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequest.java
@@ -21,6 +21,7 @@
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CLIENT_ID;
+import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CLIENT_VERSION;
 import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CURSOR;
 import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.MODE;
 
@@ -42,6 +43,7 @@ public class SqlClearCursorRequest extends AbstractSqlRequest {
         PARSER.declareString(constructorArg(), CURSOR);
         PARSER.declareString(optionalConstructorArg(), MODE);
         PARSER.declareString(optionalConstructorArg(), CLIENT_ID);
+        PARSER.declareString(optionalConstructorArg(), CLIENT_VERSION);
     }
 
     private String cursor;
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
index a338410b7278e..4329c5101c8ee 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
@@ -72,7 +72,7 @@ public SqlQueryRequest(String query, List<SqlTypedParamValue> params, QueryBuild
 
     @Override
     public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
+        ActionRequestValidationException validationException = super.validate();
         if ((false == Strings.hasText(query())) && Strings.hasText(cursor) == false) {
             validationException = addValidationError("one of [query] or [cursor] is required", validationException);
         }
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java
index f2ede4b463cdf..2e3f39c8ff449 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java
@@ -52,6 +52,11 @@ public SqlQueryRequestBuilder mode(Mode mode) {
         return this;
     }
 
+    public SqlQueryRequestBuilder version(String version) {
+        request.version(version);
+        return this;
+    }
+
     public SqlQueryRequestBuilder cursor(String cursor) {
         request.cursor(cursor);
         return this;
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java
index 708779ec6e3c3..5b4a4227980dc 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java
@@ -44,7 +44,7 @@ public SqlTranslateRequest(StreamInput in) throws IOException {
 
     @Override
     public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
+        ActionRequestValidationException validationException = super.validate();
         if ((false == Strings.hasText(query()))) {
             validationException = addValidationError("query is required", validationException);
         }
@@ -65,12 +65,12 @@ public static SqlTranslateRequest fromXContent(XContentParser parser) {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         // This is needed just to test parsing of SqlTranslateRequest, so we can reuse SqlQuerySerialization
-        return new SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(), pageTimeout(), 
-            filter(), 
-            null, 
-            null, 
+        return new SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(), pageTimeout(),
+            filter(),
+            null,
+            null,
             requestInfo(),
-            false, 
+            false,
             false,
             null).toXContent(builder, params);
     }
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
index d902e75121c75..020d0c18d52ac 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
@@ -176,6 +176,9 @@ private static void toXContent(SqlQueryRequest request, XContentBuilder builder)
         if (request.clientId() != null) {
             builder.field("client_id", request.clientId());
         }
+        if (request.version() != null) {
+            builder.field("version", request.version().toString());
+        }
         if (request.params() != null && request.params().isEmpty() == false) {
             builder.startArray("params");
             for (SqlTypedParamValue val : request.params()) {
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
index 34245945e506b..4211324e41491 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.sql.action;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -48,9 +49,10 @@ public void testClearCursorRequestParser() throws IOException {
         Mode randomMode = randomFrom(Mode.values());
         
         SqlClearCursorRequest request = generateRequest("{\"cursor\" : \"whatever\", \"mode\" : \""
-                + randomMode.toString() + "\", \"client_id\" : \"bla\"}",
+                + randomMode.toString() + "\", \"client_id\" : \"bla\", \"version\": \"1.2.3\"}",
                 SqlClearCursorRequest::fromXContent);
         assertNull(request.clientId());
+        assertNull(request.version());
         assertEquals(randomMode, request.mode());
         assertEquals("whatever", request.getCursor());
         
@@ -66,10 +68,11 @@ public void testClearCursorRequestParser() throws IOException {
         assertNull(request.clientId());
         assertEquals(Mode.PLAIN, request.mode());
         assertEquals("whatever", request.getCursor());
-        
-        request = generateRequest("{\"cursor\" : \"whatever\", \"client_id\" : \"CLI\"}",
+
+        request = generateRequest("{\"cursor\" : \"whatever\", \"client_id\" : \"CLI\", \"version\": \"1.2.3\"}",
                 SqlClearCursorRequest::fromXContent);
         assertNull(request.clientId());
+        assertNull(request.version());
         assertEquals(Mode.PLAIN, request.mode());
         assertEquals("whatever", request.getCursor());
         
@@ -101,6 +104,8 @@ public void testQueryRequestParser() throws IOException {
                 SqlQueryRequest::fromXContent);
         assertParsingErrorMessage("{\"client_id\":123}", "client_id doesn't support values of type: VALUE_NUMBER",
                 SqlQueryRequest::fromXContent);
+        assertParsingErrorMessage("{\"version\":123}", "version doesn't support values of type: VALUE_NUMBER",
+            SqlQueryRequest::fromXContent);
         assertParsingErrorMessage("{\"params\":[{\"value\":123}]}", "failed to parse field [params]", SqlQueryRequest::fromXContent);
         assertParsingErrorMessage("{\"time_zone\":12}", "time_zone doesn't support values of type: VALUE_NUMBER",
                 SqlQueryRequest::fromXContent);
@@ -108,7 +113,10 @@ public void testQueryRequestParser() throws IOException {
         Mode randomMode = randomFrom(Mode.values());
         String params;
         List<SqlTypedParamValue> list = new ArrayList<>(1);
-        
+
+        final String clientVersion = Mode.isDedicatedClient(randomMode)
+            ? "\"version\": \"" + Version.CURRENT.toString() + "\","
+            : "";
         if (Mode.isDriver(randomMode)) {
             params = "{\"value\":123, \"type\":\"whatever\"}";
             list.add(new SqlTypedParamValue("whatever", 123, true));
@@ -119,12 +127,16 @@ public void testQueryRequestParser() throws IOException {
         
         SqlQueryRequest request = generateRequest("{\"cursor\" : \"whatever\", \"mode\" : \""
                 + randomMode.toString() + "\", \"client_id\" : \"bla\","
+                + clientVersion
                 + "\"query\":\"select\","
                 + "\"params\":[" + params + "],"
                 + " \"time_zone\":\"UTC\","
                 + "\"request_timeout\":\"5s\",\"page_timeout\":\"10s\"}", SqlQueryRequest::fromXContent);
         assertNull(request.clientId());
         assertEquals(randomMode, request.mode());
+        if (Mode.isDedicatedClient(randomMode)) {
+            assertEquals(Version.CURRENT.toString(), request.version().toString());
+        }
         assertEquals("whatever", request.cursor());
         assertEquals("select", request.query());
 
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java
index c07a7f63031a8..ebf1883802515 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.xpack.sql.cli.command.ServerInfoCliCommand;
 import org.elasticsearch.xpack.sql.cli.command.ServerQueryCliCommand;
 import org.elasticsearch.xpack.sql.client.ClientException;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.ConnectionConfiguration;
 import org.elasticsearch.xpack.sql.client.HttpClient;
-import org.elasticsearch.xpack.sql.client.Version;
 import org.jline.terminal.TerminalBuilder;
 
 import java.io.IOException;
@@ -161,7 +161,7 @@ private void checkConnection(CliSession cliSession, CliTerminal cliTerminal, Con
                 // Most likely we connected to something other than Elasticsearch
                 throw new UserException(ExitCodes.DATA_ERROR,
                         "Cannot communicate with the server " + con.connectionString() +
-                                ". This version of CLI only works with Elasticsearch version " + Version.CURRENT.toString());
+                                ". This version of CLI only works with Elasticsearch version " + ClientVersion.CURRENT.toString());
             }
         }
     }
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java
index 678fc8a9c6efb..78d126f44b5bb 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/CliSession.java
@@ -5,11 +5,12 @@
  */
 package org.elasticsearch.xpack.sql.cli.command;
 
-import org.elasticsearch.xpack.sql.client.HttpClient;
 import org.elasticsearch.xpack.sql.client.ClientException;
-import org.elasticsearch.xpack.sql.client.Version;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
+import org.elasticsearch.xpack.sql.client.HttpClient;
 import org.elasticsearch.xpack.sql.proto.MainResponse;
 import org.elasticsearch.xpack.sql.proto.Protocol;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.sql.SQLException;
 
@@ -73,11 +74,11 @@ public void checkConnection() throws ClientException {
         } catch (SQLException ex) {
             throw new ClientException(ex);
         }
-        Version version = Version.fromString(response.getVersion());
-        // TODO: We can relax compatibility requirement later when we have a better idea about protocol compatibility guarantees
-        if (version.major != Version.CURRENT.major || version.minor != Version.CURRENT.minor) {
-            throw new ClientException("This alpha version of CLI is only compatible with Elasticsearch version " +
-                    Version.CURRENT.toString());
+        SqlVersion version = SqlVersion.fromString(response.getVersion());
+        if (ClientVersion.isServerCompatible(version) == false) {
+            throw new ClientException("This version of the CLI is only compatible with Elasticsearch version " +
+                ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server version " +
+                version.toString());
         }
     }
 }
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
index 40cce57321d65..bad55264768eb 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
@@ -8,7 +8,7 @@
 import org.elasticsearch.xpack.sql.cli.Cli;
 import org.elasticsearch.xpack.sql.cli.CliTerminal;
 import org.elasticsearch.xpack.sql.cli.FatalCliException;
-import org.elasticsearch.xpack.sql.client.Version;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -53,9 +53,9 @@ public void printLogo(CliTerminal terminal) {
         }
 
         // print the version centered on the last line
-        char[] whitespaces = new char[(lineLength - Version.CURRENT.version.length()) / 2];
+        char[] whitespaces = new char[(lineLength - ClientVersion.CURRENT.version.length()) / 2];
         Arrays.fill(whitespaces, ' ');
-        terminal.println(new StringBuilder().append(whitespaces).append(Version.CURRENT.version).toString());
+        terminal.println(new StringBuilder().append(whitespaces).append(ClientVersion.CURRENT.version).toString());
         terminal.println();
     }
 
diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java
index 295ebe1fb0792..438f67372ff74 100644
--- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java
+++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java
@@ -9,10 +9,11 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.cli.command.CliSession;
-import org.elasticsearch.xpack.sql.client.HttpClient;
 import org.elasticsearch.xpack.sql.client.ClientException;
-import org.elasticsearch.xpack.sql.client.Version;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
+import org.elasticsearch.xpack.sql.client.HttpClient;
 import org.elasticsearch.xpack.sql.proto.MainResponse;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.sql.SQLException;
 
@@ -26,7 +27,7 @@ public class CliSessionTests extends ESTestCase {
 
     public void testProperConnection() throws Exception {
         HttpClient httpClient = mock(HttpClient.class);
-        when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5), org.elasticsearch.Version.CURRENT.toString(),
+        when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5), ClientVersion.CURRENT.toString(),
                 ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
         CliSession cliSession = new CliSession(httpClient);
         cliSession.checkConnection();
@@ -38,28 +39,43 @@ public void testConnection() throws Exception {
         HttpClient httpClient = mock(HttpClient.class);
         when(httpClient.serverInfo()).thenThrow(new SQLException("Cannot connect"));
         CliSession cliSession = new CliSession(httpClient);
-        expectThrows(ClientException.class, cliSession::checkConnection);
+        Throwable throwable = expectThrows(ClientException.class, cliSession::checkConnection);
+        assertEquals("java.sql.SQLException: Cannot connect", throwable.getMessage());
         verify(httpClient, times(1)).serverInfo();
         verifyNoMoreInteractions(httpClient);
     }
 
     public void testWrongServerVersion() throws Exception {
+        HttpClient httpClient = mock(HttpClient.class);
+        SqlVersion version = new SqlVersion((int)SqlVersion.V_7_7_0.major, SqlVersion.V_7_7_0.minor - 1, 0);
+        when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5), version.toString(),
+                ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
+        CliSession cliSession = new CliSession(httpClient);
+        Throwable throwable = expectThrows(ClientException.class, cliSession::checkConnection);
+        assertEquals("This version of the CLI is only compatible with Elasticsearch version " +
+            ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server version " + version.toString(),
+            throwable.getMessage());
+        verify(httpClient, times(1)).serverInfo();
+        verifyNoMoreInteractions(httpClient);
+    }
+
+    public void testHigherServerVersion() throws Exception {
         HttpClient httpClient = mock(HttpClient.class);
         byte minor;
         byte major;
         if (randomBoolean()) {
-            minor = Version.CURRENT.minor;
-            major = (byte) (Version.CURRENT.major + 1);
+            minor = ClientVersion.CURRENT.minor;
+            major = (byte) (ClientVersion.CURRENT.major + 1);
         } else {
-            minor = (byte) (Version.CURRENT.minor + 1);
-            major = Version.CURRENT.major;
+            minor = (byte) (ClientVersion.CURRENT.minor + 1);
+            major = ClientVersion.CURRENT.major;
 
         }
         when(httpClient.serverInfo()).thenReturn(new MainResponse(randomAlphaOfLength(5),
-                org.elasticsearch.Version.fromString(major + "." + minor + ".23").toString(),
-                ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
+            SqlVersion.fromString(major + "." + minor + ".23").toString(),
+            ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()));
         CliSession cliSession = new CliSession(httpClient);
-        expectThrows(ClientException.class, cliSession::checkConnection);
+        cliSession.checkConnection();
         verify(httpClient, times(1)).serverInfo();
         verifyNoMoreInteractions(httpClient);
     }
diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java
index 96e3e417531a7..5b8687b950468 100644
--- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java
+++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java
@@ -5,17 +5,17 @@
  */
 package org.elasticsearch.xpack.sql.cli;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.sql.client.Version;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 
 public class VersionTests extends ESTestCase {
     public void testVersionIsCurrent() {
         /* This test will only work properly in gradle because in gradle we run the tests
          * using the jar. */
-        assertNotNull(Version.CURRENT.hash);
-        assertEquals(org.elasticsearch.Version.CURRENT.major, Version.CURRENT.major);
-        assertEquals(org.elasticsearch.Version.CURRENT.minor, Version.CURRENT.minor);
-        assertEquals(org.elasticsearch.Version.CURRENT.revision, Version.CURRENT.revision);
+        assertEquals(Version.CURRENT.major, ClientVersion.CURRENT.major);
+        assertEquals(Version.CURRENT.minor, ClientVersion.CURRENT.minor);
+        assertEquals(Version.CURRENT.revision, ClientVersion.CURRENT.revision);
     }
 
 }
diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
index 39c48bc31d926..e12889a264959 100644
--- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
+++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
@@ -7,8 +7,8 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.cli.TestTerminal;
+import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.client.HttpClient;
-import org.elasticsearch.xpack.sql.client.Version;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.mockito.Mockito.mock;
@@ -94,7 +94,7 @@ public void testPrintLogo() throws Exception {
         testTerminal.print("not clean");
         assertTrue(new PrintLogoCommand().handle(testTerminal, cliSession, "logo"));
         assertThat(testTerminal.toString(), containsString("SQL"));
-        assertThat(testTerminal.toString(), containsString(Version.CURRENT.version));
+        assertThat(testTerminal.toString(), containsString(ClientVersion.CURRENT.version));
         verifyNoMoreInteractions(httpClient);
     }
 
diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/Version.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java
similarity index 57%
rename from x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/Version.java
rename to x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java
index 93db1b506369f..6b48dc97f2c9c 100644
--- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/Version.java
+++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java
@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.sql.client;
 
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
+
 import java.io.IOException;
 import java.net.URL;
 import java.net.URLConnection;
@@ -16,45 +18,25 @@
 import java.util.jar.JarInputStream;
 import java.util.jar.Manifest;
 
-public class Version {
-
-    public static final Version CURRENT;
-    public final String version;
-    public final String hash;
-    public final byte major;
-    public final byte minor;
-    public final byte revision;
-
-    private Version(String version, String hash, byte... parts) {
-        this.version = version;
-        this.hash = hash;
-        this.major = parts[0];
-        this.minor = parts[1];
-        this.revision = parts[2];
-    }
-
-    public static Version fromString(String version) {
-        return new Version(version, "Unknown", from(version));
-    }
+/**
+ * Clients-specific version utility class.
+ * <p>
+ *     The class provides the SQL clients the version identifying the release they're are part of. The version is read from the
+ *     encompassing JAR file (Elasticsearch-specific attribute in the manifest).
+ *     The class is also a provider for the implemented JDBC standard.
+ * </p>
+ */
+public class ClientVersion {
 
-    static byte[] from(String ver) {
-        String[] parts = ver.split("[.-]");
-        // Allow for optional snapshot and qualifier
-        if (parts.length < 3 || parts.length > 5) {
-            throw new IllegalArgumentException("Invalid version " + ver);
-        }
-        else {
-            return new byte[] { Byte.parseByte(parts[0]), Byte.parseByte(parts[1]), Byte.parseByte(parts[2]) };
-        }
-    }
+    public static final SqlVersion CURRENT;
 
     static {
         // check classpath
-        String target = Version.class.getName().replace(".", "/").concat(".class");
+        String target = ClientVersion.class.getName().replace(".", "/").concat(".class");
         Enumeration<URL> res;
 
         try {
-            res = Version.class.getClassLoader().getResources(target);
+            res = ClientVersion.class.getClassLoader().getResources(target);
         } catch (IOException ex) {
             throw new IllegalArgumentException("Cannot detect Elasticsearch JDBC jar; it typically indicates a deployment issue...");
         }
@@ -85,41 +67,35 @@ static byte[] from(String ver) {
         }
 
         // This is similar to how Elasticsearch's Build class digs up its build information.
-        // Since version info is not critical, the parsing is lenient
-        URL url = Version.class.getProtectionDomain().getCodeSource().getLocation();
+        URL url = SqlVersion.class.getProtectionDomain().getCodeSource().getLocation();
         CURRENT = extractVersion(url);
     }
 
-    static Version extractVersion(URL url) {
+    static SqlVersion extractVersion(URL url) {
         String urlStr = url.toString();
-        byte maj = 0, min = 0, rev = 0;
-        String ver = "Unknown";
-        String hash = ver;
-        
         if (urlStr.endsWith(".jar") || urlStr.endsWith(".jar!/")) {
             try {
                 URLConnection conn = url.openConnection();
                 conn.setUseCaches(false);
-                
+
                 try (JarInputStream jar = new JarInputStream(conn.getInputStream())) {
                     Manifest manifest = jar.getManifest();
-                    hash = manifest.getMainAttributes().getValue("Change");
-                    ver = manifest.getMainAttributes().getValue("X-Compile-Elasticsearch-Version");
-                    byte[] vers = from(ver);
-                    maj = vers[0];
-                    min = vers[1];
-                    rev = vers[2];
+                    String version = manifest.getMainAttributes().getValue("X-Compile-Elasticsearch-Version");
+                    return SqlVersion.fromString(version);
                 }
             } catch (Exception ex) {
                 throw new IllegalArgumentException("Detected Elasticsearch JDBC jar but cannot retrieve its version", ex);
             }
         }
-        return new Version(ver, hash, maj, min, rev);
+        return new SqlVersion(0, 0, 0);
     }
 
-    @Override
-    public String toString() {
-        return "v" + version + " [" + hash + "]";
+    // This function helps ensure that a client won't attempt to communicate to a server with less features than its own. Since this check
+    // is part of the client's start-up check that might not involve an actual SQL API request, the client has to do a bare version check
+    // as well.
+    public static boolean isServerCompatible(SqlVersion server) {
+        // Starting with this version, the compatibility logic moved from the client to the server.
+        return SqlVersion.hasVersionCompatibility(server);
     }
 
     public static int jdbcMajorVersion() {
diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java
index 9d3a50b57576d..18de5d4eb4e05 100644
--- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java
+++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java
@@ -57,8 +57,6 @@ public class ConnectionConfiguration {
 
     public static final String PAGE_SIZE = "page.size";
     private static final String PAGE_SIZE_DEFAULT = "1000";
-    
-    public static final String CLIENT_ID = "client_id";
 
     // Auth
 
diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java
index 32630abb94370..2b2eee541472f 100644
--- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java
+++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java
@@ -70,7 +70,7 @@ public SqlQueryResponse basicQuery(String query, int fetchSize) throws SQLExcept
                 null,
                 Boolean.FALSE,
                 null,
-                new RequestInfo(Mode.CLI),
+                new RequestInfo(Mode.CLI, ClientVersion.CURRENT),
                 false,
                 false,
                 cfg.binaryCommunication());
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java
index 01ca2bd0dade1..ea4d6a0834ca4 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java
@@ -158,7 +158,7 @@ private void assertBinaryRequestForDrivers(boolean isBinary, XContentType xConte
                 null,
                 randomBoolean(),
                 randomAlphaOfLength(128),
-                new RequestInfo(mode),
+                new RequestInfo(mode, ClientVersion.CURRENT),
                 randomBoolean(),
                 randomBoolean(),
                 isBinary);
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java
index 03979a0a2b4e5..dabe7cdc0a384 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/VersionTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.client;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
@@ -19,45 +20,21 @@
 import java.util.jar.Manifest;
 
 public class VersionTests extends ESTestCase {
-    public void test70Version() {
-        byte[] ver = Version.from("7.0.0-alpha");
-        assertEquals(7, ver[0]);
-        assertEquals(0, ver[1]);
-        assertEquals(0, ver[2]);
-    }
-
-    public void test712Version() {
-        byte[] ver = Version.from("7.1.2");
-        assertEquals(7, ver[0]);
-        assertEquals(1, ver[1]);
-        assertEquals(2, ver[2]);
-    }
 
     public void testCurrent() {
-        Version ver = Version.fromString(org.elasticsearch.Version.CURRENT.toString());
-        assertEquals(org.elasticsearch.Version.CURRENT.major, ver.major);
-        assertEquals(org.elasticsearch.Version.CURRENT.minor, ver.minor);
-        assertEquals(org.elasticsearch.Version.CURRENT.revision, ver.revision);
-    }
-
-    public void testFromString() {
-        Version ver = Version.fromString("1.2.3");
-        assertEquals(1, ver.major);
-        assertEquals(2, ver.minor);
-        assertEquals(3, ver.revision);
-        assertEquals("Unknown", ver.hash);
-        assertEquals("1.2.3", ver.version);
+        SqlVersion ver = SqlVersion.fromString(org.elasticsearch.Version.CURRENT.toString());
+        assertEquals(ver, ClientVersion.CURRENT);
     }
 
     public void testInvalidVersion() {
-        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> Version.from("7.1"));
-        assertEquals("Invalid version 7.1", err.getMessage());
+        IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> SqlVersion.fromString("7.1"));
+        assertEquals("Invalid version format [7.1]", err.getMessage());
     }
-    
+
     public void testVersionFromJarInJar() throws IOException {
         final String JDBC_JAR_NAME = "es-sql-jdbc.jar";
         final String JAR_PATH_SEPARATOR = "!/";
-        
+
         Path dir = createTempDir();
         Path jarPath = dir.resolve("uberjar.jar");          // simulated uberjar containing the jdbc driver
         Path innerJarPath = dir.resolve(JDBC_JAR_NAME); // simulated ES JDBC driver file
@@ -88,12 +65,11 @@ public void testVersionFromJarInJar() throws IOException {
         }
         
         URL jarInJar = new URL("jar:" + jarPath.toUri().toURL().toString() + JAR_PATH_SEPARATOR + JDBC_JAR_NAME + JAR_PATH_SEPARATOR);
-        
-        Version version = Version.extractVersion(jarInJar);
+
+        SqlVersion version = ClientVersion.extractVersion(jarInJar);
         assertEquals(1, version.major);
         assertEquals(2, version.minor);
         assertEquals(3, version.revision);
-        assertEquals("abc", version.hash);
         assertEquals("1.2.3", version.version);
     }
 }
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java
index 9e8cc2fb08e5f..c6c9b8dffe0b7 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/AbstractSqlRequest.java
@@ -34,6 +34,10 @@ public String clientId() {
         return requestInfo.clientId();
     }
 
+    public SqlVersion version() {
+        return requestInfo.version();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java
index 26eb4867b3509..c5b8faed398ef 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Mode.java
@@ -33,4 +33,8 @@ public String toString() {
     public static boolean isDriver(Mode mode) {
         return mode == JDBC || mode == ODBC;
     }
+
+    public static boolean isDedicatedClient(Mode mode) {
+        return mode == JDBC || mode == ODBC || mode == CLI;
+    }
 }
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java
index 97a3622a64f1f..d26569684e391 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/RequestInfo.java
@@ -35,14 +35,25 @@ public class RequestInfo {
     
     private Mode mode;
     private String clientId;
-    
+    private SqlVersion version;
+
     public RequestInfo(Mode mode) {
-        this(mode, null);
+        this(mode, null, null);
     }
     
     public RequestInfo(Mode mode, String clientId) {
+        this(mode, clientId, null);
+    }
+
+    public RequestInfo(Mode mode, String clientId, String version) {
         mode(mode);
         clientId(clientId);
+        version(version);
+    }
+
+    public RequestInfo(Mode mode, SqlVersion version) {
+        mode(mode);
+        this.version = version;
     }
     
     public Mode mode() {
@@ -67,6 +78,14 @@ public void clientId(String clientId) {
         this.clientId = clientId;
     }
 
+    public void version(String clientVersion) {
+        this.version = SqlVersion.fromString(clientVersion);
+    }
+
+    public SqlVersion version() {
+        return version;
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(mode, clientId);
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java
index b1f374e74a8da..7b7578320d6c4 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlClearCursorRequest.java
@@ -47,6 +47,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (clientId() != null) {
             builder.field("client_id", clientId());
         }
+        if (version() != null) {
+            builder.field("version", version().toString());
+        }
         return builder;
     }
 }
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java
index e6d42c3898a91..e3d35ad2a9dbf 100644
--- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java
@@ -180,6 +180,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (clientId() != null) {
             builder.field("client_id", clientId());
         }
+        if (version() != null) {
+            builder.field("version", version().toString());
+        }
         if (this.params != null && this.params.isEmpty() == false) {
             builder.startArray("params");
             for (SqlTypedParamValue val : this.params) {
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java
new file mode 100644
index 0000000000000..4b311a373aa2a
--- /dev/null
+++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.proto;
+
+import java.security.InvalidParameterException;
+
+/**
+ * Elasticsearch's version modeler for the SQL plugin.
+ * <p>
+ *     The class models the version that the Elasticsearch server and the clients use to identify their release by. It is similar to
+ *     server's <strong>Version</strong> class (which is unavailable in this package), specific to the SQL plugin and its clients, and an
+ *     aid to establish the compatibility between them.
+ * </p>
+ */
+public class SqlVersion implements Comparable<SqlVersion>{
+
+    public final int id;
+    public final String version; // originally provided String representation
+    public final byte major;
+    public final byte minor;
+    public final byte revision;
+
+    public static final int REVISION_MULTIPLIER = 100;
+    public static final int MINOR_MULTIPLIER = REVISION_MULTIPLIER * REVISION_MULTIPLIER;
+    public static final int MAJOR_MULTIPLIER = REVISION_MULTIPLIER * MINOR_MULTIPLIER;
+
+    public static final SqlVersion V_7_7_0 = new SqlVersion(7, 7, 0);
+
+    public SqlVersion(byte major, byte minor, byte revision) {
+        this(toString(major, minor, revision), major, minor, revision);
+    }
+
+    public SqlVersion(Integer major, Integer minor, Integer revision) {
+        this(major.byteValue(), minor.byteValue(), revision.byteValue());
+        if (major > Byte.MAX_VALUE || minor > Byte.MAX_VALUE || revision > Byte.MAX_VALUE) {
+            throw new InvalidParameterException("Invalid version initialisers [" + major + ", " + minor + ", " + revision + "]");
+        }
+    }
+
+    protected SqlVersion(String version, byte... parts) {
+        this.version = version;
+
+        assert parts.length >= 3 : "Version must be initialized with all Major.Minor.Revision components";
+        this.major = parts[0];
+        this.minor = parts[1];
+        this.revision = parts[2];
+
+        if ((major | minor | revision) < 0 || minor >= REVISION_MULTIPLIER || revision >= REVISION_MULTIPLIER) {
+            throw new InvalidParameterException("Invalid version initialisers [" + major + ", " + minor + ", " + revision + "]");
+        }
+
+        id = Integer.valueOf(major) * MAJOR_MULTIPLIER
+            + Integer.valueOf(minor) * MINOR_MULTIPLIER
+            + Integer.valueOf(revision) * REVISION_MULTIPLIER;
+    }
+
+    public static SqlVersion fromString(String version) {
+        if (version == null || version.isEmpty()) {
+            return null;
+        }
+        return new SqlVersion(version, from(version));
+    }
+
+    protected static byte[] from(String ver) {
+        String[] parts = ver.split("[.-]");
+        // Allow for optional snapshot and qualifier (Major.Minor.Revision-Qualifier-SNAPSHOT)
+        if (parts.length >= 3 && parts.length <= 5) {
+            try {
+                return new byte[] { Byte.parseByte(parts[0]), Byte.parseByte(parts[1]), Byte.parseByte(parts[2]) };
+            } catch (NumberFormatException nfe) {
+                throw new IllegalArgumentException("Invalid version format [" + ver + "]: " + nfe.getMessage());
+            }
+        } else {
+            throw new IllegalArgumentException("Invalid version format [" + ver + "]");
+        }
+
+    }
+
+    private static String toString(byte... parts) {
+        assert parts.length >= 1 : "Version must contain at least a Major component";
+        String ver = String.valueOf(parts[0]);
+        for (int i = 1; i < parts.length; i ++) {
+            ver += "." + parts[i];
+        }
+        return ver;
+    }
+
+    @Override
+    public String toString() {
+        return toString(major, minor, revision);
+    }
+
+    public String majorMinorToString() {
+        return toString(major, minor);
+    }
+
+    @Override
+    public int hashCode() {
+        return id;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null) {
+            return false;
+        }
+        if (o.getClass() == getClass()) {
+            return ((SqlVersion) o).id == id;
+        }
+        if (o.getClass() == String.class) {
+            try {
+                SqlVersion v = fromString((String) o);
+                return this.equals(v);
+            } catch (IllegalArgumentException e) {
+                return false;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public int compareTo(SqlVersion o) {
+        return id - o.id;
+    }
+
+    public static int majorMinorId(SqlVersion v) {
+        return v.major * MAJOR_MULTIPLIER + v.minor * MINOR_MULTIPLIER;
+    }
+
+    public int compareToMajorMinor(SqlVersion o) {
+        return majorMinorId(this) - majorMinorId(o);
+    }
+
+    public static boolean hasVersionCompatibility(SqlVersion version) {
+        return version.compareTo(V_7_7_0) >= 0;
+    }
+}
diff --git a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java
new file mode 100644
index 0000000000000..de8d8bb6ec0ea
--- /dev/null
+++ b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.proto;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class SqlVersionTests extends ESTestCase {
+    public void test123FromString() {
+        SqlVersion ver = SqlVersion.fromString("1.2.3");
+        assertEquals(1, ver.major);
+        assertEquals(2, ver.minor);
+        assertEquals(3, ver.revision);
+        assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id);
+        assertEquals("1.2.3", ver.version);
+    }
+
+    public void test123AlphaFromString() {
+        SqlVersion ver = SqlVersion.fromString("1.2.3-Alpha");
+        assertEquals(1, ver.major);
+        assertEquals(2, ver.minor);
+        assertEquals(3, ver.revision);
+        assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id);
+        assertEquals("1.2.3-Alpha", ver.version);
+    }
+
+    public void test123AlphaSnapshotFromString() {
+        SqlVersion ver = SqlVersion.fromString("1.2.3-Alpha-SNAPSHOT");
+        assertEquals(1, ver.major);
+        assertEquals(2, ver.minor);
+        assertEquals(3, ver.revision);
+        assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id);
+        assertEquals("1.2.3-Alpha-SNAPSHOT", ver.version);
+    }
+
+    public void testVersionsEqual() {
+        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
+        SqlVersion ver2 = SqlVersion.fromString("1.2.3");
+        assertEquals(ver1, ver2);
+    }
+
+    public void testVersionsAndStringEqual() {
+        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
+        String ver2 = "1.2.3";
+        assertEquals(ver1, ver2);
+    }
+
+    public void testVersionsAndStringNotEqual() {
+        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
+        String ver2 = "1.2.4";
+        assertNotEquals(ver1, ver2);
+    }
+
+    public void testVersionsAndInvalidStringNotEqual() {
+        SqlVersion ver1 = SqlVersion.fromString("1.2.3");
+        String ver2 = "invalid";
+        assertNotEquals(ver1, ver2);
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
index 0455039a34e25..671d90fbd54f6 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
@@ -235,7 +236,7 @@ public Expression visitPredicated(PredicatedContext ctx) {
                 e = new Like(source, exp, visitPattern(pCtx.pattern()));
                 break;
             case SqlBaseParser.RLIKE:
-                e = new RLike(source, exp, string(pCtx.regex));
+                e = new RLike(source, exp, new RLikePattern(string(pCtx.regex)));
                 break;
             case SqlBaseParser.NULL:
                 // shortcut to avoid double negation later on (since there's no IsNull (missing in ES is a negated exists))
@@ -263,6 +264,9 @@ public LikePattern visitPattern(PatternContext ctx) {
         }
 
         String pattern = string(ctx.value);
+        if (pattern == null) {
+            throw new ParsingException(source(ctx.value), "Pattern must not be [null]");
+        }
         int pos = pattern.indexOf('*');
         if (pos >= 0) {
             throw new ParsingException(source(ctx.value),
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
index 412f5475fbfc2..8a3453cda5509 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
@@ -60,7 +60,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
          */
         String accept = null;
 
-        if ((Mode.isDriver(sqlRequest.requestInfo().mode()) || sqlRequest.requestInfo().mode() == Mode.CLI)
+        if (Mode.isDedicatedClient(sqlRequest.requestInfo().mode())
                 && (sqlRequest.binaryCommunication() == null || sqlRequest.binaryCommunication())) {
             // enforce CBOR response for drivers and CLI (unless instructed differently through the config param)
             accept = XContentType.CBOR.name();
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
index e958ce1aa830d..bb5156a47b35b 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
@@ -36,6 +36,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.TO_NULL;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -159,7 +160,7 @@ public static Converter converterFor(DataType from, DataType to) {
         }
         // extend the default converter with DATE and TIME
         if (from == DATE || from == TIME) {
-            if (to == KEYWORD || to == TEXT) {
+            if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
                 return conversionToString(from);
             }
             if (to == LONG) {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
index 6c321bea80de8..616b2de40810a 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
@@ -29,6 +29,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -268,6 +269,7 @@ public static boolean isFromDocValuesOnly(DataType dataType) {
                 || dataType == DATE         // because of date formats
                 || dataType == DATETIME
                 || dataType == SCALED_FLOAT // because of scaling_factor
+                || dataType == CONSTANT_KEYWORD
                 || dataType == GEO_POINT
                 || dataType == GEO_SHAPE
                 || dataType == SHAPE;
@@ -333,6 +335,9 @@ public static SQLType sqlType(DataType dataType) {
         if (dataType == TEXT) {
             return JDBCType.VARCHAR;
         }
+        if (dataType == CONSTANT_KEYWORD) {
+            return JDBCType.VARCHAR;
+        }
         if (dataType == DATETIME) {
             return JDBCType.TIMESTAMP;
         }
@@ -456,6 +461,9 @@ public static int defaultPrecision(DataType dataType) {
         if (dataType == TEXT) {
             return 32766;
         }
+        if (dataType == CONSTANT_KEYWORD) {
+            return 15;
+        }
         if (dataType == DATETIME) {
             return 3;
         }
@@ -576,6 +584,9 @@ public static int displaySize(DataType dataType) {
         if (dataType == TEXT) {
             return dataType.size();
         }
+        if (dataType == CONSTANT_KEYWORD) {
+            return 32766;
+        }
         if (dataType == DATETIME) {
             return 29;
         }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java
index 5f85ff90e344c..f5a834acade80 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java
@@ -5,12 +5,14 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.xpack.sql.proto.ColumnInfo;
 import org.elasticsearch.xpack.sql.proto.Mode;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertRequestBuilderThrows;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -28,7 +30,7 @@ public void testSqlAction() {
         boolean dataBeforeCount = randomBoolean();
         String columns = dataBeforeCount ? "data, count" : "count, data";
         SqlQueryResponse response = new SqlQueryRequestBuilder(client(), SqlQueryAction.INSTANCE)
-                .query("SELECT " + columns + " FROM test ORDER BY count").mode(Mode.JDBC).get();
+                .query("SELECT " + columns + " FROM test ORDER BY count").mode(Mode.JDBC).version(Version.CURRENT.toString()).get();
         assertThat(response.size(), equalTo(2L));
         assertThat(response.columns(), hasSize(2));
         int dataIndex = dataBeforeCount ? 0 : 1;
@@ -42,5 +44,18 @@ public void testSqlAction() {
         assertEquals("baz", response.rows().get(1).get(dataIndex));
         assertEquals(43L, response.rows().get(1).get(countIndex));
     }
+
+    public void testSqlActionCurrentVersion() {
+        SqlQueryResponse response = new SqlQueryRequestBuilder(client(), SqlQueryAction.INSTANCE)
+            .query("SELECT true").mode(randomFrom(Mode.CLI, Mode.JDBC)).version(Version.CURRENT.toString()).get();
+        assertThat(response.size(), equalTo(1L));
+        assertEquals(true, response.rows().get(0).get(0));
+    }
+
+    public void testSqlActionOutdatedVersion() {
+        SqlQueryRequestBuilder request = new SqlQueryRequestBuilder(client(), SqlQueryAction.INSTANCE)
+            .query("SELECT true").mode(randomFrom(Mode.CLI, Mode.JDBC)).version("1.2.3");
+        assertRequestBuilderThrows(request, org.elasticsearch.action.ActionRequestValidationException.class);
+    }
 }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
index b53d1cbdf8133..614cc24ec44aa 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
@@ -10,10 +10,12 @@
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
+import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.ql.type.InvalidMappedField;
 import org.elasticsearch.xpack.ql.type.KeywordEsField;
+import org.elasticsearch.xpack.ql.type.TextEsField;
 import org.elasticsearch.xpack.sql.type.SqlDataTypeRegistry;
 
 import java.util.ArrayList;
@@ -26,6 +28,7 @@
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
@@ -268,6 +271,52 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
                 ((InvalidMappedField) esField).errorMessage());
     }
 
+    public void testConstantKeywordTwoIndicesScenario() throws Exception {
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "name", "text", true, false);
+        addFieldCaps(fieldCaps, "wheels_count", "constant_keyword", true, true);
+        addFieldCaps(fieldCaps, "motor", "keyword", true, true);
+
+        Map<String, FieldCapabilities> multi = new HashMap<>();
+        multi.put("keyword", new FieldCapabilities("cycle_type", "keyword", true, true, new String[] { "other_cycles" }, null, null,
+                Collections.emptyMap()));
+        multi.put("constant_keyword", new FieldCapabilities("cycle_type", "constant_keyword", true, false, new String[] { "bicycles" },
+                null, null, Collections.emptyMap()));
+        fieldCaps.put("cycle_type", multi);
+
+        String wildcard = "*";
+        IndexResolution resolution = mergedMappings(wildcard, new String[] { "other_cycles", "bicycles" }, fieldCaps);
+
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        EsField esField = null;
+        Map<String, EsField> props = esIndex.mapping();
+
+        assertEquals(4, props.size());
+        assertTrue(props.containsKey("cycle_type"));
+        assertTrue(props.containsKey("name"));
+        assertTrue(props.containsKey("wheels_count"));
+        assertTrue(props.containsKey("motor"));
+        
+        esField = props.get("cycle_type");
+        assertTrue(esField instanceof KeywordEsField);
+        assertEquals(KEYWORD, ((KeywordEsField) esField).getDataType());
+        
+        esField = props.get("name");
+        assertTrue(esField instanceof TextEsField);
+        assertEquals(TEXT, ((TextEsField) esField).getDataType());
+        
+        esField = props.get("wheels_count");
+        assertTrue(esField instanceof ConstantKeywordEsField);
+        assertEquals(CONSTANT_KEYWORD, ((ConstantKeywordEsField) esField).getDataType());
+        
+        esField = props.get("motor");
+        assertTrue(esField instanceof KeywordEsField);
+        assertEquals(KEYWORD, ((KeywordEsField) esField).getDataType());
+    }
+
     public void testSeparateSameMappingDifferentIndices() throws Exception {
         Map<String, EsField> oneMapping = loadMapping("mapping-basic.json", true);
         Map<String, EsField> sameMapping = loadMapping("mapping-basic.json", true);
@@ -383,7 +432,7 @@ private static boolean isSearchable(DataType type) {
     }
 
     private static boolean isAggregatable(DataType type) {
-        return type.isNumeric() || type == KEYWORD || type == DATETIME;
+        return type.isNumeric() || type == KEYWORD || type == DATETIME || type == CONSTANT_KEYWORD;
     }
 
     private static class UpdateableFieldCapabilities extends FieldCapabilities {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
index d4ef29f667ee0..922b32d199f7f 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
@@ -369,7 +370,7 @@ public void testGenericNullableExpression() {
         // comparison
         assertNullLiteral(rule.rule(new GreaterThan(EMPTY, getFieldAttribute(), NULL)));
         // regex
-        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, "123")));
+        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123"))));
     }
 
     public void testNullFoldingOnCast() {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
index ad2d3066e121f..6211f40e0f0e5 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
@@ -21,16 +21,19 @@
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Sub;
+import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue;
 
 import java.time.Duration;
 import java.time.Period;
 import java.time.temporal.TemporalAmount;
+import java.util.Collections;
 import java.util.Locale;
 
 import static java.lang.String.format;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.LONG;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -542,4 +545,11 @@ public void testCaseWithOperand() {
         assertEquals("WHEN 1 THEN 'one'", ifc.sourceText());
         assertEquals("many", c.elseResult().toString());
     }
+
+    public void testLikePatternWithNullParameterNotAllowed() {
+        ParsingException e = expectThrows(ParsingException.class,
+                () -> parser.createExpression("a LIKE ?",
+                    Collections.singletonList(new SqlTypedParamValue(KEYWORD.typeName(), null))));
+        assertEquals("line 1:9: Pattern must not be [null]", e.getMessage());
+    }
 }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
index 17bc17bbec4b8..e0bf79b830e00 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
@@ -46,7 +46,7 @@ public void testSysTypes() {
         Command cmd = sql("SYS TYPES").v1();
 
         List<String> names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", "SCALED_FLOAT",
-                "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME",
+                "CONSTANT_KEYWORD", "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME",
                 "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND",
                 "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND",
                 "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND",
@@ -107,7 +107,9 @@ public void testSysTypesMultipleMatches() {
 
         cmd.execute(session(), wrap(p -> {
             SchemaRowSet r = (SchemaRowSet) p.rowSet();
-            assertEquals(3, r.size());
+            assertEquals(4, r.size());
+            assertEquals("CONSTANT_KEYWORD", r.column(0));
+            assertTrue(r.advanceRow());
             assertEquals("IP", r.column(0));
             assertTrue(r.advanceRow());
             assertEquals("KEYWORD", r.column(0));
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
index a70a0d610c36b..2225ead0dfd31 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
@@ -547,6 +547,23 @@ private void assertDifferentRLikeAndNotRLikePatterns(String firstPattern, String
         assertEquals("keyword", rqsq.field());
     }
 
+    public void testLikeRLikeAsPainlessScripts() {
+        LogicalPlan p = plan("SELECT count(*), CASE WHEN keyword LIKE '%foo%' THEN 1 WHEN keyword RLIKE '.*bar.*' THEN 2 " +
+                             "ELSE 3 END AS t FROM test GROUP BY t");
+        assertTrue(p instanceof Aggregate);
+        Expression condition = ((Aggregate) p).groupings().get(0);
+        assertFalse(condition.foldable());
+        GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
+        assertNotNull(groupingContext);
+        ScriptTemplate scriptTemplate = groupingContext.tail.script();
+        assertEquals("InternalSqlScriptUtils.caseFunction([InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
+                     "doc,params.v0),params.v1),params.v2,InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
+                     "doc,params.v3),params.v4),params.v5,params.v6])",
+                scriptTemplate.toString());
+        assertEquals("[{v=keyword}, {v=^.*foo.*$}, {v=1}, {v=keyword}, {v=.*bar.*}, {v=2}, {v=3}]",
+                scriptTemplate.params().toString());
+    }
+
     public void testTranslateNotExpression_WhereClause_Painless() {
         LogicalPlan p = plan("SELECT * FROM test WHERE NOT(POSITION('x', keyword) = 0)");
         assertTrue(p instanceof Project);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
index b39dd8bcaa897..c445a84b236ca 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
@@ -22,6 +22,7 @@
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -631,6 +632,7 @@ public void testCommonType() {
         assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
         assertEquals(NULL, commonType(NULL, NULL));
         assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
+        assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
         assertEquals(LONG, commonType(TEXT, LONG));
         assertEquals(SHORT, commonType(SHORT, BYTE));
         assertEquals(FLOAT, commonType(BYTE, FLOAT));
@@ -640,6 +642,11 @@ public void testCommonType() {
         // strings
         assertEquals(TEXT, commonType(TEXT, KEYWORD));
         assertEquals(TEXT, commonType(KEYWORD, TEXT));
+        assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
+        assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
+        assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
+        assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
+        assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
 
         // numeric and intervals
         assertEquals(INTERVAL_YEAR_TO_MONTH, commonType(INTERVAL_YEAR_TO_MONTH, LONG));
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
new file mode 100644
index 0000000000000..ab9171b7bc064
--- /dev/null
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
@@ -0,0 +1,127 @@
+/*
+ *
+ *  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ *  * or more contributor license agreements. Licensed under the Elastic License;
+ *  * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+
+package org.elasticsearch.xpack.test.rest;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.matchesRegex;
+
+public class CatIndicesWithSecurityIT extends ESRestTestCase {
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("cat_test_user", new SecureString("cat-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    public void testHiddenIndexWithVisibleAlias() throws IOException {
+        // Create the index and alias
+        {
+            final Request createRequest = new Request("PUT", ".index_hidden");
+            createRequest.setJsonEntity(
+                "{\"settings\": {\"index.hidden\": true, \"number_of_replicas\":  0}, \"aliases\": {\"index_allowed\": {}}}");
+            final Response createResponse = adminClient().performRequest(createRequest);
+            assertOK(createResponse);
+            ensureGreen("index_allowed");
+        }
+
+        // And it should be visible by a user that has permissions for only the alias
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
+        }
+    }
+
+    public void testHiddenIndexWithHiddenAlias() throws IOException {
+        // Create the index and alias
+        {
+            final Request createRequest = new Request("PUT", ".index_hidden");
+            createRequest.setJsonEntity("{\"settings\": {\"index.hidden\": true, \"number_of_replicas\":  0}, "
+                + "\"aliases\": {\"index_allowed\": {\"is_hidden\":  true}}}");
+            final Response createResponse = adminClient().performRequest(createRequest);
+            assertOK(createResponse);
+            ensureGreen("index_allowed");
+        }
+
+        // It should *not* be visible
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
+        }
+
+        // But we should be able to see the index if we ask for hidden things
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
+        }
+    }
+
+    public void testVisibleIndexWithHiddenAlias() throws IOException {
+        // Create the index and alias
+        {
+            final Request createRequest = new Request("PUT", "visible_index");
+            createRequest.setJsonEntity("{\"settings\": {\"number_of_replicas\":  0}, "
+                + "\"aliases\": {\"index_allowed\": {\"is_hidden\":  true}}}");
+            final Response createResponse = adminClient().performRequest(createRequest);
+            assertOK(createResponse);
+            ensureGreen("index_allowed");
+        }
+
+        // It should *not* be visible, because this user only has access through the *hidden* alias
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
+        }
+
+        // But we should be able to see the index if we ask for hidden things
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+visible_index.*$"));
+        }
+    }
+}
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
index 3d5d92c8eb753..dd207033554e8 100644
--- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.CheckedFunction;
@@ -24,6 +25,7 @@
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
+import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.junit.After;
 import org.junit.Before;
 
@@ -75,7 +77,7 @@ public void setupForTests() throws Exception {
     }
 
     /**
-     * Waits for the Security template and the Machine Learning templates to be created by the {@link MetaDataUpgrader}
+     * Waits for Machine Learning and Transform templates to be created by the {@link MetaDataUpgrader}
      */
     private void waitForTemplates() throws Exception {
         if (installTemplates()) {
@@ -86,7 +88,10 @@ private void waitForTemplates() throws Exception {
                     MlMetaIndex.INDEX_NAME,
                     AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
                     AnomalyDetectorsIndex.jobResultsIndexPrefix(),
-                    AnomalyDetectorsIndex.configIndexName()));
+                    AnomalyDetectorsIndex.configIndexName(),
+                    TransformInternalIndexConstants.AUDIT_INDEX,
+                    TransformInternalIndexConstants.LATEST_INDEX_NAME
+                ));
 
             for (String template : templates) {
                 awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(),
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
new file mode 100644
index 0000000000000..4486b90cb9a1d
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
@@ -0,0 +1,24 @@
+{
+  "async_search.delete":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_async_search/{id}",
+          "methods":[
+            "DELETE"
+          ],
+          "parts":{
+            "id":{
+              "type":"string",
+              "description":"The async search ID"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
new file mode 100644
index 0000000000000..f5ea1424756e1
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
@@ -0,0 +1,40 @@
+{
+  "async_search.get":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_async_search/{id}",
+          "methods":[
+            "GET"
+          ],
+          "parts":{
+            "id":{
+              "type":"string",
+              "description":"The async search ID"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "wait_for_completion":{
+        "type":"time",
+        "description":"Specify the time that the request should block waiting for the final response",
+        "default": "1s"
+      },
+      "keep_alive": {
+        "type": "time",
+        "description": "Specify the time interval in which the results (partial or final) for this search will be available",
+        "default": "5d"
+      },
+      "typed_keys":{
+        "type":"boolean",
+        "description":"Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
new file mode 100644
index 0000000000000..033939acf0703
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
@@ -0,0 +1,225 @@
+{
+  "async_search.submit":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_async_search",
+          "methods":[
+            "POST"
+          ]
+        },
+        {
+          "path":"/{index}/_async_search",
+          "methods":[
+            "POST"
+          ],
+          "parts":{
+            "index":{
+              "type":"list",
+              "description":"A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "wait_for_completion":{
+        "type":"time",
+        "description":"Specify the time that the request should block waiting for the final response",
+        "default": "1s"
+      },
+      "keep_alive": {
+        "type": "time",
+        "description": "Update the time interval in which the results (partial or final) for this search will be available"
+      },
+      "batched_reduce_size":{
+        "type":"number",
+        "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.",
+        "default":5
+      },
+      "analyzer":{
+        "type":"string",
+        "description":"The analyzer to use for the query string"
+      },
+      "analyze_wildcard":{
+        "type":"boolean",
+        "description":"Specify whether wildcard and prefix queries should be analyzed (default: false)"
+      },
+      "default_operator":{
+        "type":"enum",
+        "options":[
+          "AND",
+          "OR"
+        ],
+        "default":"OR",
+        "description":"The default operator for query string query (AND or OR)"
+      },
+      "df":{
+        "type":"string",
+        "description":"The field to use as default where no field prefix is given in the query string"
+      },
+      "explain":{
+        "type":"boolean",
+        "description":"Specify whether to return detailed information about score computation as part of a hit"
+      },
+      "stored_fields":{
+        "type":"list",
+        "description":"A comma-separated list of stored fields to return as part of a hit"
+      },
+      "docvalue_fields":{
+        "type":"list",
+        "description":"A comma-separated list of fields to return as the docvalue representation of a field for each hit"
+      },
+      "from":{
+        "type":"number",
+        "description":"Starting offset (default: 0)"
+      },
+      "ignore_unavailable":{
+        "type":"boolean",
+        "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+      },
+      "ignore_throttled":{
+        "type":"boolean",
+        "description":"Whether specified concrete, expanded or aliased indices should be ignored when throttled"
+      },
+      "allow_no_indices":{
+        "type":"boolean",
+        "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+      },
+      "expand_wildcards":{
+        "type":"enum",
+        "options":[
+          "open",
+          "closed",
+          "none",
+          "all"
+        ],
+        "default":"open",
+        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
+      },
+      "lenient":{
+        "type":"boolean",
+        "description":"Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
+      },
+      "preference":{
+        "type":"string",
+        "description":"Specify the node or shard the operation should be performed on (default: random)"
+      },
+      "q":{
+        "type":"string",
+        "description":"Query in the Lucene query string syntax"
+      },
+      "routing":{
+        "type":"list",
+        "description":"A comma-separated list of specific routing values"
+      },
+      "search_type":{
+        "type":"enum",
+        "options":[
+          "query_then_fetch",
+          "dfs_query_then_fetch"
+        ],
+        "description":"Search operation type"
+      },
+      "size":{
+        "type":"number",
+        "description":"Number of hits to return (default: 10)"
+      },
+      "sort":{
+        "type":"list",
+        "description":"A comma-separated list of <field>:<direction> pairs"
+      },
+      "_source":{
+        "type":"list",
+        "description":"True or false to return the _source field or not, or a list of fields to return"
+      },
+      "_source_excludes":{
+        "type":"list",
+        "description":"A list of fields to exclude from the returned _source field"
+      },
+      "_source_includes":{
+        "type":"list",
+        "description":"A list of fields to extract and return from the _source field"
+      },
+      "terminate_after":{
+        "type":"number",
+        "description":"The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early."
+      },
+      "stats":{
+        "type":"list",
+        "description":"Specific 'tag' of the request for logging and statistical purposes"
+      },
+      "suggest_field":{
+        "type":"string",
+        "description":"Specify which field to use for suggestions"
+      },
+      "suggest_mode":{
+        "type":"enum",
+        "options":[
+          "missing",
+          "popular",
+          "always"
+        ],
+        "default":"missing",
+        "description":"Specify suggest mode"
+      },
+      "suggest_size":{
+        "type":"number",
+        "description":"How many suggestions to return in response"
+      },
+      "suggest_text":{
+        "type":"string",
+        "description":"The source text for which the suggestions should be returned"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      },
+      "track_scores":{
+        "type":"boolean",
+        "description":"Whether to calculate and return scores even if they are not used for sorting"
+      },
+      "track_total_hits":{
+        "type":"boolean",
+        "description":"Indicate if the number of documents that match the query should be tracked"
+      },
+      "allow_partial_search_results":{
+        "type":"boolean",
+        "default":true,
+        "description":"Indicate if an error should be returned if there is a partial search failure or timeout"
+      },
+      "typed_keys":{
+        "type":"boolean",
+        "description":"Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
+      },
+      "version":{
+        "type":"boolean",
+        "description":"Specify whether to return document version as part of a hit"
+      },
+      "seq_no_primary_term":{
+        "type":"boolean",
+        "description":"Specify whether to return sequence number and primary term of the last modification of each hit"
+      },
+      "request_cache":{
+        "type":"boolean",
+        "description":"Specify if request cache should be used for this request or not, defaults to index level setting"
+      },
+      "max_concurrent_shard_requests":{
+        "type":"number",
+        "description":"The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests",
+        "default":5
+      },
+      "clean_on_completion":{
+        "type":"boolean",
+        "description":"Control whether the response should not be stored in the cluster if it completed within the provided [wait_for_completion] time (default: true)"
+      }
+    },
+    "body":{
+      "description":"The search definition using the Query DSL"
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json
new file mode 100644
index 0000000000000..6eaa707a4969f
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.estimate_model_memory.json
@@ -0,0 +1,23 @@
+{
+  "ml.estimate_model_memory":{
+    "documentation":{
+      "url":null
+    },
+    "stability":"stable",
+    "url":{
+      "paths":[
+        {
+          "path":"/_ml/anomaly_detectors/_estimate_model_memory",
+          "methods":[
+            "POST"
+          ]
+        }
+      ]
+    },
+    "params":{},
+    "body":{
+      "description":"The analysis config, plus cardinality estimates for fields it references",
+      "required":true
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
new file mode 100644
index 0000000000000..ef457a569ad0e
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
@@ -0,0 +1,143 @@
+---
+"Async search":
+  - do:
+      indices.create:
+        index: test-1
+        body:
+          settings:
+            number_of_shards: "2"
+
+  - do:
+      indices.create:
+        index: test-2
+        body:
+          settings:
+            number_of_shards: "1"
+
+  - do:
+      indices.create:
+        index: test-3
+        body:
+          settings:
+            number_of_shards: "3"
+
+  - do:
+      index:
+        index:  test-2
+        body:   { max: 2 }
+
+  - do:
+      index:
+        index:  test-1
+        body:   { max: 1 }
+
+  - do:
+      index:
+        index:  test-3
+        body:   { max: 3 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      async_search.submit:
+        index: test-*
+        batched_reduce_size: 2
+        wait_for_completion: 10s
+        body:
+          query:
+            match_all: {}
+          aggs:
+            1:
+              max:
+                field: max
+          sort: max
+
+  - is_false: id
+  - match:  { version:                          6 }
+  - match:  { is_partial:                   false }
+  - length: { response.hits.hits:               3 }
+  - match:  { response.hits.hits.0._source.max: 1 }
+  - match:  { response.aggregations.1.value:  3.0 }
+
+  - do:
+      async_search.submit:
+        index: test-*
+        batched_reduce_size: 2
+        wait_for_completion: 10s
+        clean_on_completion: false
+        body:
+          aggs:
+            1:
+              max:
+                field: max
+          sort: max
+
+  - set:    { id:                              id }
+  - match:  { version:                          6 }
+  - match:  { is_partial:                   false }
+  - length: { response.hits.hits:               3 }
+  - match:  { response.hits.hits.0._source.max: 1 }
+  - match:  { response.aggregations.1.value:  3.0 }
+
+  # test with typed_keys:
+  - do:
+      async_search.submit:
+        index: test-*
+        batched_reduce_size: 2
+        wait_for_completion: 10s
+        clean_on_completion: false
+        typed_keys: true
+        body:
+          aggs:
+            1:
+              max:
+                field: max
+          sort: max
+
+  - set:    { id:                                  id }
+  - match:  { version:                              6 }
+  - match:  { is_partial:                       false }
+  - length: { response.hits.hits:                   3 }
+  - match:  { response.hits.hits.0._source.max:     1 }
+  - match:  { response.aggregations.max#1.value:  3.0 }
+
+  - do:
+      async_search.get:
+        id: "$id"
+
+  - match:  { version:                            6 }
+  - match:  { is_partial:                     false }
+  - length: { response.hits.hits:                 3 }
+  - match:  { response.hits.hits.0._source.max:   1 }
+  - match:  { response.aggregations.1.value:    3.0 }
+
+  # test with typed_keys:
+  - do:
+      async_search.get:
+        id: "$id"
+        typed_keys: true
+
+  - match:  { version:                              6 }
+  - match:  { is_partial:                       false }
+  - length: { response.hits.hits:                   3 }
+  - match:  { response.hits.hits.0._source.max:     1 }
+  - match:  { response.aggregations.max#1.value:  3.0 }
+
+  - do:
+      async_search.delete:
+        id: "$id"
+
+  - match: { acknowledged:   true }
+
+  - do:
+      catch: missing
+      async_search.get:
+        id: "$id"
+
+  - do:
+      catch: missing
+      async_search.delete:
+        id: "$id"
+
+
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
index df394a85f0995..a5a99b30391ea 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -1333,12 +1333,12 @@ setup:
           }
 
 ---
-"Test put regression given maximum_number_trees is zero":
+"Test put regression given max_trees is zero":
 
   - do:
-      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "regression-maximum-number-trees-is-zero"
+        id: "regression-max-trees-is-zero"
         body: >
           {
             "source": {
@@ -1350,18 +1350,18 @@ setup:
             "analysis": {
               "regression": {
                 "dependent_variable": "foo",
-                "maximum_number_trees": 0
+                "max_trees": 0
               }
             }
           }
 
 ---
-"Test put regression given maximum_number_trees is greater than 2k":
+"Test put regression given max_trees is greater than 2k":
 
   - do:
-      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "regression-maximum-number-trees-greater-than-2k"
+        id: "regression-max-trees-greater-than-2k"
         body: >
           {
             "source": {
@@ -1373,7 +1373,7 @@ setup:
             "analysis": {
               "regression": {
                 "dependent_variable": "foo",
-                "maximum_number_trees": 2001
+                "max_trees": 2001
               }
             }
           }
@@ -1490,7 +1490,7 @@ setup:
                 "lambda": 3.14,
                 "gamma": 0.42,
                 "eta": 0.5,
-                "maximum_number_trees": 400,
+                "max_trees": 400,
                 "feature_bag_fraction": 0.3,
                 "training_percent": 60.3,
                 "randomize_seed": 42
@@ -1506,7 +1506,7 @@ setup:
       "lambda": 3.14,
       "gamma": 0.42,
       "eta": 0.5,
-      "maximum_number_trees": 400,
+      "max_trees": 400,
       "feature_bag_fraction": 0.3,
       "prediction_field_name": "foo_prediction",
       "training_percent": 60.3,
@@ -1629,12 +1629,12 @@ setup:
           }
 
 ---
-"Test put classification given maximum_number_trees is zero":
+"Test put classification given max_trees is zero":
 
   - do:
-      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "classification-maximum-number-trees-is-zero"
+        id: "classification-max-trees-is-zero"
         body: >
           {
             "source": {
@@ -1646,18 +1646,18 @@ setup:
             "analysis": {
               "classification": {
                 "dependent_variable": "foo",
-                "maximum_number_trees": 0
+                "max_trees": 0
               }
             }
           }
 
 ---
-"Test put classification given maximum_number_trees is greater than 2k":
+"Test put classification given max_trees is greater than 2k":
 
   - do:
-      catch: /\[maximum_number_trees\] must be an integer in \[1, 2000\]/
+      catch: /\[max_trees\] must be an integer in \[1, 2000\]/
       ml.put_data_frame_analytics:
-        id: "classification-maximum-number-trees-greater-than-2k"
+        id: "classification-max-trees-greater-than-2k"
         body: >
           {
             "source": {
@@ -1669,7 +1669,7 @@ setup:
             "analysis": {
               "classification": {
                 "dependent_variable": "foo",
-                "maximum_number_trees": 2001
+                "max_trees": 2001
               }
             }
           }
@@ -1832,8 +1832,9 @@ setup:
                 "lambda": 3.14,
                 "gamma": 0.42,
                 "eta": 0.5,
-                "maximum_number_trees": 400,
+                "max_trees": 400,
                 "feature_bag_fraction": 0.3,
+                "class_assignment_objective": "maximize_accuracy",
                 "training_percent": 60.3,
                 "randomize_seed": 24
               }
@@ -1848,11 +1849,56 @@ setup:
       "lambda": 3.14,
       "gamma": 0.42,
       "eta": 0.5,
-      "maximum_number_trees": 400,
+      "max_trees": 400,
       "feature_bag_fraction": 0.3,
       "prediction_field_name": "foo_prediction",
       "training_percent": 60.3,
       "randomize_seed": 24,
+      "class_assignment_objective": "maximize_accuracy",
+      "num_top_classes": 2
+    }
+  }}
+  - is_true: create_time
+  - is_true: version
+
+---
+"Test put classification given deprecated maximum_number_trees":
+
+  - skip:
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - 'Deprecated field [maximum_number_trees] used, expected [max_trees] instead'
+      ml.put_data_frame_analytics:
+        id: "classification-with-maximum-number-trees"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {
+              "classification": {
+                "dependent_variable": "foo",
+                "maximum_number_trees": 400,
+                "randomize_seed": 24
+              }
+            }
+          }
+  - match: { id: "classification-with-maximum-number-trees" }
+  - match: { source.index: ["index-source"] }
+  - match: { dest.index: "index-dest" }
+  - match: { analysis: {
+    "classification":{
+      "dependent_variable": "foo",
+      "max_trees": 400,
+      "prediction_field_name": "foo_prediction",
+      "training_percent": 100.0,
+      "randomize_seed": 24,
+      "class_assignment_objective": "maximize_minimum_recall",
       "num_top_classes": 2
     }
   }}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
new file mode 100644
index 0000000000000..8253f12dda70e
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
@@ -0,0 +1,172 @@
+---
+"Test by field":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline"}]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            }
+          }
+  - match: { model_memory_estimate: "3135mb" }
+
+---
+"Test by field also influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline"}],
+              "influencers": [ "airline" ]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            },
+            "max_bucket_cardinality": {
+              "airline": 500
+            }
+          }
+  - match: { model_memory_estimate: "3135mb" }
+
+---
+"Test by field with independent influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline"}],
+              "influencers": [ "country" ]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            },
+            "max_bucket_cardinality": {
+              "country": 500
+            }
+          }
+  - match: { model_memory_estimate: "3140mb" }
+
+---
+"Test partition field":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "partition_field_name": "airline"}]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            }
+          }
+  - match: { model_memory_estimate: "3135mb" }
+
+---
+"Test partition field also influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "partition_field_name": "airline"}],
+              "influencers": [ "airline" ]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            },
+            "max_bucket_cardinality": {
+              "airline": 500
+            }
+          }
+  - match: { model_memory_estimate: "3135mb" }
+
+---
+"Test partition field with independent influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "partition_field_name": "airline"}],
+              "influencers": [ "country" ]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            },
+            "max_bucket_cardinality": {
+              "country": 500
+            }
+          }
+  - match: { model_memory_estimate: "3140mb" }
+
+---
+"Test by and partition field":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline", "partition_field_name": "country"}]
+            },
+            "overall_cardinality": {
+              "airline": 4000,
+              "country": 600
+            }
+          }
+  - match: { model_memory_estimate: "150010mb" }
+
+---
+"Test by and partition fields also influencers":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline", "partition_field_name": "country"}],
+              "influencers": [ "airline", "country" ]
+            },
+            "overall_cardinality": {
+              "airline": 4000,
+              "country": 600
+            },
+            "max_bucket_cardinality": {
+              "airline": 60,
+              "country": 40
+            }
+          }
+  - match: { model_memory_estimate: "150010mb" }
+
+---
+"Test by and partition fields with independent influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "by_field_name": "airline", "partition_field_name": "country"}],
+              "influencers": [ "src_ip" ]
+            },
+            "overall_cardinality": {
+              "airline": 4000,
+              "country": 600
+            },
+            "max_bucket_cardinality": {
+              "src_ip": 500
+            }
+          }
+  - match: { model_memory_estimate: "150015mb" }
+
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
index 28ef49182361f..6774ae8b455cd 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
@@ -111,7 +111,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       headers:
@@ -464,7 +464,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       headers:
@@ -514,7 +514,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       ml.delete_job:
@@ -607,7 +607,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       ml.delete_job:
@@ -685,7 +685,7 @@ setup:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       headers:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
index cb5ccd90e1311..8c1f1e581de5b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
@@ -313,7 +313,7 @@ setup:
                   "model_id" : "a-regression-model-0",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
new file mode 100644
index 0000000000000..9c043eee3cd2c
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
@@ -0,0 +1,114 @@
+# These tests are not part of the ml-with-security tests
+# These are more like ingest processor tests than actual ML focused tests.
+setup:
+  - skip:
+      features: headers
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      ml.put_trained_model:
+        model_id: a-perfect-regression-model
+        body: >
+          {
+            "description": "empty model for tests",
+            "tags": ["regression", "tag1"],
+            "input": {"field_names": ["field1", "field2"]},
+            "definition": {
+               "preprocessors": [],
+               "trained_model": {
+                  "tree": {
+                     "feature_names": ["field1", "field2"],
+                     "tree_structure": [
+                        {"node_index": 0, "leaf_value": 42}
+                     ],
+                     "target_type": "regression"
+                  }
+               }
+            }
+          }
+
+---
+"Test create and delete pipeline with inference processor":
+  - do:
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "inference_config": {"regression": {}},
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+  - do:
+      ingest.delete_pipeline:
+        id: "regression-model-pipeline"
+---
+"Test create processor with missing mandatory fields":
+  - do:
+      catch: /\[model_id\] required property is missing/
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "inference_config": {"regression": {}},
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+  - do:
+      catch: /\[inference_config\] required property is missing/
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+---
+"Test create processor with deprecated fields":
+  - skip:
+      features:
+       - "warnings"
+       - "allowed_warnings"
+  - do:
+      warnings:
+        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "inference_config": {"regression": {}},
+                  "field_mappings": {}
+                }
+              }
+            ]
+          }
+
+  - do:
+      allowed_warnings:
+        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
+      ingest.delete_pipeline:
+        id: "regression-model-pipeline"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
index 5143a690b018a..40d0f6ba01ab7 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
@@ -66,7 +66,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]
@@ -84,7 +84,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
index a5242b5062e1f..445ab11b04288 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
@@ -471,7 +471,7 @@
 
   - do:
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       catch: /Invalid update value for analysis_limits[:] model_memory_limit cannot be decreased below current usage; current usage \[9mb\], update had \[5mb\]/
@@ -928,7 +928,7 @@
 
   - do:
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       catch: /status_exception/
@@ -956,7 +956,7 @@
 
   - do:
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       catch: /status_exception/
@@ -994,7 +994,7 @@
 
   - do:
       indices.refresh:
-        index: ["*",".ml-*"]
+        expand_wildcards: all
 
   - do:
       catch: /status_exception/
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
index 81077b1e69a8c..ab6cfac9515e5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
@@ -142,7 +142,7 @@
         id: "start_given_empty_dest_index"
 
 ---
-"Test start classification analysis when the dependent variable cardinality is too low or too high":
+"Test start classification analysis when the dependent variable cardinality is too low":
   - do:
       indices.create:
         index: index-with-dep-var-with-too-high-card
@@ -179,22 +179,3 @@
       catch: /Field \[keyword_field\] must have at least \[2\] distinct values but there were \[1\]/
       ml.start_data_frame_analytics:
         id: "classification-cardinality-limits"
-
-  - do:
-      index:
-        index: index-with-dep-var-with-too-high-card
-        body: { numeric_field: 2.0, keyword_field: "class_b" }
-
-  - do:
-      index:
-        index: index-with-dep-var-with-too-high-card
-        body: { numeric_field: 3.0, keyword_field: "class_c" }
-
-  - do:
-      indices.refresh:
-        index: index-with-dep-var-with-too-high-card
-
-  - do:
-      catch: /Field \[keyword_field\] must have at most \[2\] distinct values but there were at least \[3\]/
-      ml.start_data_frame_analytics:
-        id: "classification-cardinality-limits"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
new file mode 100644
index 0000000000000..c67a79c8218da
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
@@ -0,0 +1,218 @@
+setup:
+  - skip:
+      features: headers
+      version: " - 7.9.99"
+      reason: "wildcard fields were added from 8.0"
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            number_of_replicas: 0
+          mappings:
+            properties:
+              my_wildcard:
+                 type: wildcard
+  - do:
+      index:
+        index: test-index
+        id: 1
+        body:
+          my_wildcard: hello world
+  - do:
+      index:
+        index: test-index
+        id: 2
+        body:
+          my_wildcard: goodbye world
+
+  - do:
+      indices.refresh: {}
+
+---
+"Short prefix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hel*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Long prefix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hello wor*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Short unrooted query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Long unrooted query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello worl*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Short suffix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ld" }
+
+
+  - match: {hits.total.value: 2}
+
+---
+"Long suffix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello world" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"No wildcard wildcard query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hello world" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Term query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            term:
+              my_wildcard: "hello world"
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Terms query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            terms:
+              my_wildcard: ["hello world", "does not exist"]
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Prefix query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            prefix:
+              my_wildcard:
+                value: "hell*"
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Sequence fail":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*world*hello*" }
+
+
+  - match: {hits.total.value: 0}
+
+---
+"Aggs work":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*world*" }
+          aggs:
+            top_vals:
+              terms: {field: "my_wildcard" }
+
+
+  - match: {hits.total.value: 2}
+  - length: { aggregations.top_vals.buckets: 2 }
+
+---
+"Sort works":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          sort: [ { "my_wildcard": "desc" } ]
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.1._id: "2" }
+
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          sort: [ { "my_wildcard": "asc" } ]
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.1._id: "1" }
+
+
diff --git a/x-pack/plugin/src/test/resources/roles.yml b/x-pack/plugin/src/test/resources/roles.yml
new file mode 100644
index 0000000000000..5f8677860d079
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/roles.yml
@@ -0,0 +1,7 @@
+# Required for CatIndicesWithSecurityIT
+cat_test_role:
+  cluster:
+    - monitor
+  indices:
+    - names: [ "index_allowed" ]
+      privileges: [ "read", "write", "monitor" ]
diff --git a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
index 96dde81bf8a45..7dd064f7dbc91 100644
--- a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
+++ b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
@@ -14,11 +14,11 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -54,7 +54,10 @@ public final void startWatcher() throws Exception {
                     throw new AssertionError("waiting until starting state reached started state");
                 case "started":
                     int watcherCount = (int) response.evaluate("stats.0.watch_count");
-                    assertThat(watcherCount, equalTo(0));
+                    if (watcherCount > 0) {
+                        logger.info("expected 0 active watches, but got [{}], deleting watcher indices again", watcherCount);
+                        deleteWatcherIndices();
+                    }
                     // all good here, we are done
                     break;
                 default:
@@ -88,7 +91,10 @@ public final void stopWatcher() throws Exception {
                     throw new AssertionError("unknown state[" + state + "]");
             }
         }, 60, TimeUnit.SECONDS);
+        deleteWatcherIndices();
+    }
 
+    private static void deleteWatcherIndices() throws IOException {
         Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
         deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
         adminClient().performRequest(deleteWatchesIndexRequest);
diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle
new file mode 100644
index 0000000000000..22e29aa0832f2
--- /dev/null
+++ b/x-pack/plugin/wildcard/build.gradle
@@ -0,0 +1,18 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'wildcard'
+  description 'A plugin for a keyword field type with efficient wildcard search'
+  classname 'org.elasticsearch.xpack.wildcard.Wildcard'
+  extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-wildcard'
+
+dependencies {
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+integTest.enabled = false
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
new file mode 100644
index 0000000000000..3749dc2622c8b
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.wildcard;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.wildcard.mapper.WildcardFieldMapper;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class Wildcard extends Plugin implements MapperPlugin {
+
+
+    public Wildcard(Settings settings) {
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
+        mappers.put(WildcardFieldMapper.CONTENT_TYPE, new WildcardFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(mappers);
+    }
+}
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
new file mode 100644
index 0000000000000..648fbc7e0cdc3
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.ByteRunAutomaton;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Query that runs an Automaton across all binary doc values. 
+ * Expensive to run so normally used in conjunction with more selective query clauses.
+ */
+public class AutomatonQueryOnBinaryDv extends Query {
+
+    private final String field;
+    private final String matchPattern;
+    private final Automaton automaton;
+
+    public AutomatonQueryOnBinaryDv(String field, String matchPattern, Automaton automaton) {
+        this.field = field;
+        this.matchPattern = matchPattern;
+        this.automaton = automaton;
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+                
+        ByteRunAutomaton bytesMatcher = new ByteRunAutomaton(automaton);
+        
+        return new ConstantScoreWeight(this, boost) {
+
+            @Override
+            public Scorer scorer(LeafReaderContext context) throws IOException {
+                ByteArrayDataInput badi = new ByteArrayDataInput();
+                final BinaryDocValues values = DocValues.getBinary(context.reader(), field);               
+                TwoPhaseIterator twoPhase = new TwoPhaseIterator(values) {
+                    @Override
+                    public boolean matches() throws IOException {
+                        BytesRef arrayOfValues = values.binaryValue();
+                        badi.reset(arrayOfValues.bytes);
+                        badi.setPosition(arrayOfValues.offset);
+                        
+                        int size = badi.readVInt();
+                        for (int i=0; i< size; i++) {
+                            int valLength = badi.readVInt();
+                            if (bytesMatcher.run(arrayOfValues.bytes, badi.getPosition(), valLength)) {
+                                return true;
+                            }
+                            badi.skipBytes(valLength);
+                        }
+                        return false;
+                    }
+
+                    @Override
+                    public float matchCost() {
+                        // TODO: how can we compute this?
+                        return 1000f;
+                    }
+                };
+                return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
+            }
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                return true;
+            }
+        };
+    }
+    @Override
+    public String toString(String field) {
+        return field+":"+matchPattern;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        AutomatonQueryOnBinaryDv other = (AutomatonQueryOnBinaryDv) obj;
+        return Objects.equals(field, other.field)  && Objects.equals(matchPattern, other.matchPattern);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, matchPattern);
+    }
+
+}
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
new file mode 100644
index 0000000000000..e489d8a35bb9f
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -0,0 +1,575 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ngram.NGramTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.MultiTermQuery.RewriteMethod;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.automaton.Automaton;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
+import org.elasticsearch.index.mapper.BinaryFieldMapper.CustomBinaryDocValuesField;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.similarity.SimilarityProvider;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.index.mapper.TypeParsers.parseField;
+
+/**
+ * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching
+ */
+public class WildcardFieldMapper extends FieldMapper {
+
+    public static final String CONTENT_TYPE = "wildcard";
+    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; 
+    public static final int NGRAM_SIZE = 3;        
+    static final NamedAnalyzer WILDCARD_ANALYZER = new NamedAnalyzer("_wildcard", AnalyzerScope.GLOBAL, new Analyzer() {
+        @Override
+        public TokenStreamComponents createComponents(String fieldName) {
+            Tokenizer tokenizer = new NGramTokenizer(NGRAM_SIZE, NGRAM_SIZE);
+            return new TokenStreamComponents(tokenizer);
+        }
+    });    
+
+    public static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new WildcardFieldType();
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setIndexAnalyzer(WILDCARD_ANALYZER);
+            FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.setStoreTermVectorOffsets(false);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.freeze();
+        }
+        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;        
+    }
+
+    public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
+        protected int ignoreAbove = Defaults.IGNORE_ABOVE;
+
+        public Builder(String name) {
+            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        @Override
+        public Builder docValues(boolean docValues) {
+            if (docValues == false) {
+                throw new MapperParsingException("The field [" + name + "] cannot have doc values = false");                
+            }
+            return this;
+        }
+
+        @Override
+        public Builder indexOptions(IndexOptions indexOptions) {
+            if (indexOptions != IndexOptions.DOCS) {
+                throw new MapperParsingException("The field [" + name + "] cannot have indexOptions = " + indexOptions);
+            }
+            return this;
+        }
+
+        @Override
+        public Builder store(boolean store) {
+            if (store) {
+                throw new MapperParsingException("The field [" + name + "] cannot have store = true");                
+            }
+            return this;
+        }
+
+        @Override
+        public Builder similarity(SimilarityProvider similarity) {
+            throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");                
+        }
+
+        @Override
+        public Builder index(boolean index) {
+            if (index == false) {
+                throw new MapperParsingException("The field [" + name + "] cannot have index = false");                
+            }
+            return this;
+        }
+        
+        public Builder ignoreAbove(int ignoreAbove) {
+            if (ignoreAbove < 0) {
+                throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
+            }
+            this.ignoreAbove = ignoreAbove;
+            return this;
+        }        
+          
+        
+        @Override
+        protected void setupFieldType(BuilderContext context) {
+            super.setupFieldType(context);
+            fieldType().setHasDocValues(true);
+            fieldType().setTokenized(false);
+            fieldType().setIndexOptions(IndexOptions.DOCS);                
+        }
+
+        @Override
+        public WildcardFieldType fieldType() {
+            return (WildcardFieldType) super.fieldType();
+        }
+
+        @Override
+        public WildcardFieldMapper build(BuilderContext context) {
+            setupFieldType(context);            
+            return new WildcardFieldMapper(
+                    name, fieldType, defaultFieldType, ignoreAbove, 
+                    context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
+                throws MapperParsingException {
+            WildcardFieldMapper.Builder builder = new WildcardFieldMapper.Builder(name);
+            parseField(builder, name, node, parserContext);
+            
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String propName = entry.getKey();
+                Object propNode = entry.getValue();
+                if (propName.equals("ignore_above")) {
+                    builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
+                    iterator.remove();
+                }
+            }            
+            
+            return builder;
+        }
+    }
+    
+     public static final char TOKEN_START_OR_END_CHAR = 0;    
+    
+     public static final class WildcardFieldType extends MappedFieldType {
+
+        public WildcardFieldType() {            
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);            
+        }
+
+        protected WildcardFieldType(WildcardFieldType ref) {
+            super(ref);
+        }
+
+        public WildcardFieldType clone() {
+            WildcardFieldType result = new WildcardFieldType(this);
+            return result;
+        }
+        
+                
+        // Holds parsed information about the wildcard pattern
+        static class PatternStructure {
+            boolean openStart, openEnd, hasSymbols;            
+            int lastGap =0;
+            int wildcardCharCount, wildcardStringCount;
+            String[] fragments;
+            Integer []  precedingGapSizes;
+            final String pattern;
+            
+            @SuppressWarnings("fallthrough") // Intentionally uses fallthrough mirroring implementation in Lucene's WildcardQuery
+            PatternStructure (String wildcardText) {
+                this.pattern = wildcardText;
+                ArrayList<String> fragmentList = new ArrayList<>();
+                ArrayList<Integer> precedingGapSizeList = new ArrayList<>();
+                StringBuilder sb = new StringBuilder();               
+                for (int i = 0; i < wildcardText.length();) {
+                    final int c = wildcardText.codePointAt(i);
+                    int length = Character.charCount(c);
+                    switch (c) {
+                    case WildcardQuery.WILDCARD_STRING:
+                        if (i == 0) {
+                            openStart = true;
+                        }
+                        openEnd = true;                        
+                        hasSymbols = true;
+                        wildcardStringCount++;
+                        
+                        if (sb.length() > 0) {
+                            precedingGapSizeList.add(lastGap);
+                            fragmentList.add(sb.toString());
+                            sb = new StringBuilder();
+                        }
+                        lastGap = Integer.MAX_VALUE;
+                        break;
+                    case WildcardQuery.WILDCARD_CHAR:
+                        if (i == 0) {
+                            openStart = true;
+                        }
+                        hasSymbols = true;
+                        wildcardCharCount++;
+                        openEnd = true;
+                        if (sb.length() > 0) {
+                            precedingGapSizeList.add(lastGap);
+                            fragmentList.add(sb.toString());
+                            sb = new StringBuilder();       
+                            lastGap = 0;
+                        }
+                        
+                        if (lastGap != Integer.MAX_VALUE) {
+                            lastGap++;
+                        }
+                        break;
+                    case WildcardQuery.WILDCARD_ESCAPE:
+                        // add the next codepoint instead, if it exists
+                        if (i + length < wildcardText.length()) {
+                            final int nextChar = wildcardText.codePointAt(i + length);
+                            length += Character.charCount(nextChar);
+                            sb.append(Character.toChars(nextChar));
+                            openEnd = false;
+                            break;
+                        } // else fallthru, lenient parsing with a trailing \
+                    default:
+                        openEnd = false;
+                        sb.append(Character.toChars(c));
+                    }
+                    i += length;
+                }
+                if (sb.length() > 0) {
+                    precedingGapSizeList.add(lastGap);
+                    fragmentList.add(sb.toString());
+                    lastGap = 0;
+                }
+                fragments = fragmentList.toArray(new String[0]);
+                precedingGapSizes = precedingGapSizeList.toArray(new Integer[0]);
+                
+            }
+            
+            public boolean needsVerification() {
+                // Return true if term queries are not enough evidence
+                if (fragments.length == 1 && wildcardCharCount == 0) {
+                    // The one case where we don't need verification is when 
+                    // we have a single fragment and no ? characters
+                    return false;
+                }
+                return true;
+            }
+            
+            // Returns number of positions for last gap (Integer.MAX means unlimited gap)
+            public int getPrecedingGapSize(int fragmentNum) {
+                return precedingGapSizes[fragmentNum];
+            }
+
+            public boolean isMatchAll() {
+                return fragments.length == 0 && wildcardStringCount >0 && wildcardCharCount ==0;
+            }
+
+            @Override
+            public int hashCode() {
+                return pattern.hashCode();
+            }
+
+            @Override
+            public boolean equals(Object obj) {
+                PatternStructure other = (PatternStructure) obj;
+                return pattern.equals(other.pattern);
+            }
+            
+            
+        }
+        
+
+        @Override
+        public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
+            PatternStructure patternStructure = new PatternStructure(wildcardPattern);            
+            ArrayList<String> tokens = new ArrayList<>();
+
+            for (int i = 0; i < patternStructure.fragments.length; i++) {
+                String fragment = patternStructure.fragments[i];
+                int fLength = fragment.length();
+                if (fLength == 0) {
+                    continue;
+                }
+                
+                // Add any start/end of string character
+                if (i == 0 && patternStructure.openStart == false) {
+                    // Start-of-string anchored (is not a leading wildcard)
+                    fragment = TOKEN_START_OR_END_CHAR + fragment;
+                }
+                if (patternStructure.openEnd == false && i == patternStructure.fragments.length - 1) {
+                    // End-of-string anchored (is not a trailing wildcard)
+                    fragment = fragment + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
+                }
+                if (fragment.codePointCount(0, fragment.length()) <= NGRAM_SIZE) {
+                    tokens.add(fragment);
+                } else {
+                    // Break fragment into multiple Ngrams                
+                    TokenStream tokenizer = WILDCARD_ANALYZER.tokenStream(name(), fragment);
+                    CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class);
+                    String lastUnusedToken = null;
+                    try {
+                        tokenizer.reset();
+                        boolean takeThis = true;
+                        // minimise number of terms searched - eg for "12345" and 3grams we only need terms
+                        // `123` and `345` - no need to search for 234. We take every other ngram.
+                        while (tokenizer.incrementToken()) {
+                            String tokenValue = termAtt.toString();
+                            if (takeThis) {
+                                tokens.add(tokenValue);
+                            } else {
+                                lastUnusedToken = tokenValue;                                
+                            }
+                            // alternate
+                            takeThis = !takeThis;
+                        }
+                        if (lastUnusedToken != null) {
+                            // given `cake` and 3 grams the loop above would output only `cak` and we need to add trailing
+                            // `ake` to complete the logic.
+                            tokens.add(lastUnusedToken);
+                        }
+                        tokenizer.end();
+                        tokenizer.close();
+                    } catch (IOException ioe) {
+                        throw new ElasticsearchParseException("Error parsing wildcard query pattern fragment [" + fragment + "]");
+                    }
+                }
+            }
+
+            if (patternStructure.isMatchAll()) {
+                return new MatchAllDocsQuery();
+            } 
+            BooleanQuery approximation = createApproximationQuery(tokens);
+            if (approximation.clauses().size() > 1 || patternStructure.needsVerification()) {
+                BooleanQuery.Builder verifyingBuilder = new BooleanQuery.Builder();
+                verifyingBuilder.add(new BooleanClause(approximation, Occur.MUST));
+                Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), wildcardPattern));
+                verifyingBuilder.add(new BooleanClause(new AutomatonQueryOnBinaryDv(name(), wildcardPattern, automaton), Occur.MUST));
+                return verifyingBuilder.build();
+            }
+            return approximation;
+        }                
+
+        private BooleanQuery createApproximationQuery(ArrayList<String> tokens) {
+            BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();
+            if (tokens.size() <= MAX_CLAUSES_IN_APPROXIMATION_QUERY) {
+                for (String token : tokens) {
+                    addClause(token, bqBuilder);
+                }
+                return bqBuilder.build();
+            }
+            // Thin out the number of clauses using a selection spread evenly across the range
+            float step = (float) (tokens.size() - 1) / (float) (MAX_CLAUSES_IN_APPROXIMATION_QUERY - 1); // set step size
+            for (int i = 0; i < MAX_CLAUSES_IN_APPROXIMATION_QUERY; i++) {
+                addClause(tokens.get(Math.round(step * i)), bqBuilder); // add each element of a position which is a multiple of step
+            }
+            // TODO we can be smarter about pruning here. e.g.
+            // * Avoid wildcard queries if there are sufficient numbers of other terms that are full 3grams that are cheaper term queries
+            // * We can select terms on their scarcity rather than even spreads across the search string.
+            
+            return bqBuilder.build();
+        }
+
+        private void addClause(String token, BooleanQuery.Builder bqBuilder) {
+            assert token.codePointCount(0, token.length()) <= NGRAM_SIZE;
+            if (token.codePointCount(0, token.length()) == NGRAM_SIZE) {
+                TermQuery tq = new TermQuery(new Term(name(), token));
+                bqBuilder.add(new BooleanClause(tq, Occur.MUST));
+            } else {
+                WildcardQuery wq = new WildcardQuery(new Term(name(), token + "*"));
+                wq.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
+                bqBuilder.add(new BooleanClause(wq, Occur.MUST));
+            }
+
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            return new DocValuesFieldExistsQuery(name());
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, context);
+        }
+        
+        @Override
+        public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+            return wildcardQuery(value + "*", method, context);
+        }        
+
+        @Override
+        public Query termsQuery(List<?> values, QueryShardContext context) {
+            BooleanQuery.Builder bq = new BooleanQuery.Builder();
+            for (Object value : values) {
+                bq.add(termQuery(value, context), Occur.SHOULD);
+            }
+            return new ConstantScoreQuery(bq.build());
+        }     
+                
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
+            failIfNoDocValues();
+            return new IndexFieldData.Builder() {
+
+                @Override
+                public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
+                        CircuitBreakerService breakerService, MapperService mapperService) {
+                    return new WildcardBytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
+                }};
+        }        
+    }
+     
+    static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
+
+        WildcardBytesBinaryDVIndexFieldData(Index index, String fieldName) {
+            super(index, fieldName);
+        }
+
+        @Override
+        public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue,
+                    sortMode, nested);
+            return new SortField(getFieldName(), source, reverse);
+        }
+    
+    }
+
+    private int ignoreAbove;
+
+    private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+                int ignoreAbove, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
+        this.ignoreAbove = ignoreAbove;
+        assert fieldType.indexOptions() == IndexOptions.DOCS;
+        
+        ngramFieldType = fieldType.clone();
+        ngramFieldType.setTokenized(true);                    
+        ngramFieldType.freeze();
+    }
+
+    /** Values that have more chars than the return value of this method will
+     *  be skipped at parsing time. */
+    // pkg-private for testing
+    int ignoreAbove() {
+        return ignoreAbove;
+    }    
+    
+    @Override
+    protected WildcardFieldMapper clone() {
+        return (WildcardFieldMapper) super.clone();
+    }
+
+    @Override
+    public WildcardFieldType fieldType() {
+        return (WildcardFieldType) super.fieldType();
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        super.doXContentBody(builder, includeDefaults, params);
+        if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
+            builder.field("ignore_above", ignoreAbove);
+        }
+    }
+    
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        final String value;
+        if (context.externalValueSet()) {
+            value = context.externalValue().toString();
+        } else {
+            XContentParser parser = context.parser();
+            if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+                value = fieldType().nullValueAsString();
+            } else {
+                value =  parser.textOrNull();
+            }
+        }
+        ParseContext.Document parseDoc = context.doc();
+        
+        createFields(value, parseDoc, fields);        
+    }   
+    
+    // For internal use by Lucene only - used to define ngram index
+    final MappedFieldType ngramFieldType;
+    
+    void createFields(String value, Document parseDoc, List<IndexableField>fields) {
+        if (value == null || value.length() > ignoreAbove) {
+            return;
+        }
+        String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
+        Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
+        fields.add(ngramField);
+        
+        CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name());
+        if (dvField == null) {
+            dvField = new CustomBinaryDocValuesField(fieldType().name(), value.getBytes(StandardCharsets.UTF_8));
+            parseDoc.addWithKey(fieldType().name(), dvField);
+        } else {
+            dvField.add(value.getBytes(StandardCharsets.UTF_8));
+        }        
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+    
+
+    @Override
+    protected void doMerge(Mapper mergeWith) {
+        super.doMerge(mergeWith);
+        this.ignoreAbove = ((WildcardFieldMapper) mergeWith).ignoreAbove;
+    }    
+}
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
new file mode 100644
index 0000000000000..f8c8ddc7f5eef
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
@@ -0,0 +1,331 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.xpack.wildcard.mapper.WildcardFieldMapper.Builder;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.function.BiFunction;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class WildcardFieldMapperTests extends ESTestCase {
+
+    private static final String KEYWORD_FIELD_NAME = "keyword_field";
+    private static final String WILDCARD_FIELD_NAME = "wildcard_field";
+    static final int MAX_FIELD_LENGTH = 100;
+    static WildcardFieldMapper wildcardFieldType;
+    static KeywordFieldMapper keywordFieldType;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        Builder builder = new WildcardFieldMapper.Builder(WILDCARD_FIELD_NAME);
+        builder.ignoreAbove(MAX_FIELD_LENGTH);        
+        wildcardFieldType = builder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));
+        
+        
+        org.elasticsearch.index.mapper.KeywordFieldMapper.Builder kwBuilder = new KeywordFieldMapper.Builder(KEYWORD_FIELD_NAME);
+        keywordFieldType = kwBuilder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));        
+        super.setUp();
+    }
+    
+    public void testIllegalDocValuesArgument() {
+        Builder ft = new WildcardFieldMapper.Builder("test");
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+                () -> ft.docValues(false));
+        assertEquals("The field [test] cannot have doc values = false", e.getMessage());
+    }        
+
+    public void testIllegalIndexedArgument() {
+        Builder ft = new WildcardFieldMapper.Builder("test");
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+                () -> ft.index(false));
+        assertEquals("The field [test] cannot have index = false", e.getMessage());
+    }
+    
+    public void testTooBigKeywordField() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+        
+        // Create a string that is too large and will not be indexed
+        String docContent = randomABString(MAX_FIELD_LENGTH + 1);
+        Document doc = new Document();
+        ParseContext.Document parseDoc = new ParseContext.Document();                        
+        addFields(parseDoc, doc, docContent);
+        indexDoc(parseDoc, doc, iw);        
+
+        iw.forceMerge(1);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+
+        Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery("*a*", null, null);
+        TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
+        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
+                
+        reader.close();
+        dir.close();        
+    }
+    
+    //Test long query strings don't cause exceptions
+    public void testTooBigQueryField() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+        
+        // Create a string that is too large and will not be indexed
+        String docContent = randomABString(10);
+        Document doc = new Document();
+        ParseContext.Document parseDoc = new ParseContext.Document();                        
+        addFields(parseDoc, doc, docContent);
+        indexDoc(parseDoc, doc, iw);        
+
+        iw.forceMerge(1);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+
+        String queryString = randomABString((BooleanQuery.getMaxClauseCount() * 2) + 1);
+        Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(queryString, null, null);
+        TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
+        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
+                
+        reader.close();
+        dir.close();        
+    }    
+    
+    
+    public void testSearchResultsVersusKeywordField() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+        int numDocs = 100;
+        HashSet<String> values = new HashSet<>();
+        for (int i = 0; i < numDocs; i++) {
+            Document doc = new Document();
+            ParseContext.Document parseDoc = new ParseContext.Document();                        
+            String docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
+            if (values.contains(docContent) == false) {                
+                addFields(parseDoc, doc, docContent);
+                values.add(docContent);
+            }
+            // Occasionally add a multi-value field
+            if (randomBoolean()) {
+                docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
+                if (values.contains(docContent) == false) {                
+                    addFields(parseDoc, doc, docContent);
+                    values.add(docContent);
+                }                
+            }
+            indexDoc(parseDoc, doc, iw);
+            
+        }
+
+        iw.forceMerge(1);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+
+        int numSearches = 100;
+        for (int i = 0; i < numSearches; i++) {
+            String randomWildcardPattern = getRandomWildcardPattern();
+
+            Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(randomWildcardPattern, null, null);
+            TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, values.size() + 1, Sort.INDEXORDER);
+
+            Query keywordFieldQuery = new WildcardQuery(new Term(KEYWORD_FIELD_NAME, randomWildcardPattern));
+            TopDocs kwTopDocs = searcher.search(keywordFieldQuery, values.size() + 1, Sort.INDEXORDER);
+
+            assertThat(kwTopDocs.totalHits.value, equalTo(wildcardFieldTopDocs.totalHits.value));
+
+            HashSet<Integer> expectedDocs = new HashSet<>();
+            for (ScoreDoc topDoc : kwTopDocs.scoreDocs) {
+                expectedDocs.add(topDoc.doc);
+            }
+            for (ScoreDoc wcTopDoc : wildcardFieldTopDocs.scoreDocs) {
+                assertTrue(expectedDocs.remove(wcTopDoc.doc));
+            }
+            assertThat(expectedDocs.size(), equalTo(0));
+        }
+        
+        
+        //Test keyword and wildcard sort operations are also equivalent
+        QueryShardContext shardContextMock = createMockShardContext();
+        
+        FieldSortBuilder wildcardSortBuilder = new FieldSortBuilder(WILDCARD_FIELD_NAME);
+        SortField wildcardSortField = wildcardSortBuilder.build(shardContextMock).field;        
+        ScoreDoc[] wildcardHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(wildcardSortField)).scoreDocs;
+
+        FieldSortBuilder keywordSortBuilder = new FieldSortBuilder(KEYWORD_FIELD_NAME);
+        SortField keywordSortField = keywordSortBuilder.build(shardContextMock).field;        
+        ScoreDoc[] keywordHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(keywordSortField)).scoreDocs;
+        
+        assertThat(wildcardHits.length, equalTo(keywordHits.length));
+        for (int i = 0; i < wildcardHits.length; i++) {
+            assertThat(wildcardHits[i].doc, equalTo(keywordHits[i].doc));            
+        }
+        
+        reader.close();
+        dir.close();
+    }
+
+    
+    
+    protected MappedFieldType provideMappedFieldType(String name) {
+        if (name.equals(WILDCARD_FIELD_NAME)) {
+            return wildcardFieldType.fieldType();            
+        } else {
+            return keywordFieldType.fieldType();
+        }
+    }    
+    
+    protected final QueryShardContext createMockShardContext() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index,
+            Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build());
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class));
+        BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup = (fieldType, fieldIndexName) -> {
+            IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName);
+            return builder.build(idxSettings, fieldType, new IndexFieldDataCache.None(), null, null);
+        };
+        return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup,
+                null, null, null, xContentRegistry(), null, null, null,
+                () -> randomNonNegativeLong(), null, null, () -> true) {
+
+            @Override
+            public MappedFieldType fieldMapper(String name) {
+                return provideMappedFieldType(name);
+            }
+        };
+    }       
+    
+    private void addFields(ParseContext.Document parseDoc, Document doc, String docContent) throws IOException {
+        ArrayList<IndexableField> fields = new ArrayList<>();
+        wildcardFieldType.createFields(docContent, parseDoc, fields);
+
+        for (IndexableField indexableField : fields) {
+            doc.add(indexableField);
+        }
+        // Add keyword fields too
+        doc.add(new SortedSetDocValuesField(KEYWORD_FIELD_NAME, new BytesRef(docContent)));        
+        doc.add(new StringField(KEYWORD_FIELD_NAME, docContent, Field.Store.YES));
+    }
+
+    private void indexDoc(ParseContext.Document parseDoc, Document doc, RandomIndexWriter iw) throws IOException {
+        IndexableField field = parseDoc.getByKey(wildcardFieldType.name());
+        if (field != null) {
+            doc.add(field);
+        }
+        iw.addDocument(doc);
+    }
+
+    protected IndexSettings createIndexSettings() {
+        return new IndexSettings(
+                IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                        .numberOfShards(1).numberOfReplicas(0).creationDate(System.currentTimeMillis()).build(),
+                Settings.EMPTY);
+    }
+
+
+    static String randomABString(int minLength) {
+        StringBuilder sb = new StringBuilder();
+        while (sb.length() < minLength) {
+            if (randomBoolean()) {
+                sb.append("a");
+            } else {
+                sb.append("b");
+            }
+        }
+        return sb.toString();
+    }
+
+    private void randomSyntaxChar(StringBuilder sb) {
+        switch (randomInt(3)) {
+        case 0:
+            sb.append(WildcardQuery.WILDCARD_CHAR);
+            break;
+        case 1:
+            sb.append(WildcardQuery.WILDCARD_STRING);
+            break;
+        case 2:
+            sb.append(WildcardQuery.WILDCARD_ESCAPE);
+            sb.append(WildcardQuery.WILDCARD_STRING);
+            break;
+        case 3:
+            sb.append(WildcardQuery.WILDCARD_ESCAPE);
+            sb.append(WildcardQuery.WILDCARD_CHAR);
+            break;
+        }
+    }
+
+    private String getRandomWildcardPattern() {
+        StringBuilder sb = new StringBuilder();
+        int numFragments = 1 + randomInt(4);
+        if (randomInt(10) == 1) {
+            randomSyntaxChar(sb);
+        }
+        for (int i = 0; i < numFragments; i++) {
+            if (i > 0) {
+                randomSyntaxChar(sb);
+            }
+            sb.append(randomABString(1 + randomInt(6)));
+        }
+        if (randomInt(10) == 1) {
+            randomSyntaxChar(sb);
+        }
+        return sb.toString();
+    }
+}
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java
new file mode 100644
index 0000000000000..0d090e4a25058
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+public class WildcardFieldTypeTests extends FieldTypeTestCase {
+
+    @Override
+    protected MappedFieldType createDefaultFieldType() {
+        return new WildcardFieldMapper.WildcardFieldType();
+    }  
+}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
index 2dc76febf8dc4..d6651382e4e4d 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
@@ -77,7 +77,16 @@ public void testMlConfigIndexMappingsAfterMigration() throws Exception {
             // trigger .ml-config index mappings update
             createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID);
             // assert that the mappings are updated
-            assertThat(getConfigIndexMappings(), equalTo(expectedConfigIndexMappings));
+            Map<String, Object> configIndexMappings = getConfigIndexMappings();
+
+            // Remove renamed fields
+            if (getOldClusterVersion().before(Version.V_8_0_0)) {
+                configIndexMappings = XContentMapValues.filter(expectedConfigIndexMappings, null, new String[] {
+                    "analysis.properties.*.properties.maximum_number_trees" // This was renamed to max_trees
+                });
+            }
+
+            assertThat(configIndexMappings, equalTo(expectedConfigIndexMappings));
         }
     }
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index af304afc57db6..be388da19be07 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
+
 ---
 "Get old outlier_detection job":
 
@@ -92,6 +97,31 @@
   - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
   - match: { data_frame_analytics.0.state: "stopped" }
 
+---
+"Get old classification job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
+  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
+  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
+
+---
+"Get old classification job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
 ---
 "Put an outlier_detection job on the mixed cluster":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index fe160bba15f23..10ab98a738e1c 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,4 +1,7 @@
 setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
 
   - do:
       index:
@@ -19,6 +22,16 @@ setup:
             "user": "Kimchy"
           }
 
+  - do:
+      index:
+        index: bwc_ml_classification_job_source
+        body: >
+          {
+            "numeric_field_1": 1.0,
+            "foo": "a",
+            "user": "Kimchy"
+          }
+
   - do:
       indices.refresh:
         index: bwc_ml_*
@@ -64,3 +77,26 @@ setup:
             }
           }
   - match: { id: "old_cluster_regression_job" }
+
+---
+"Put classification job on the old cluster":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "old_cluster_classification_job"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_classification_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "old_cluster_classification_job_results"
+            },
+            "analysis": {
+              "classification":{
+                "dependent_variable": "foo"
+              }
+            }
+          }
+  - match: { id: "old_cluster_classification_job" }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index 14438883f0da1..aa4ef717263f9 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
+
 ---
 "Get old cluster outlier_detection job":
 
@@ -52,6 +57,32 @@
   - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
   - match: { data_frame_analytics.0.state: "stopped" }
 
+---
+"Get old classification job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
+  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
+  - match: { data_frame_analytics.0.analysis.classification.class_assignment_objective: "maximize_minimum_recall" }
+  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
+
+---
+"Get old classification job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
 ---
 "Get mixed cluster outlier_detection job":
   - skip:

From a6f0b9a3b7f611e935558290f7b3e343e8e33b5a Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 17 Mar 2020 08:57:44 +0100
Subject: [PATCH 20/51] use locale with toLowerCase

---
 .../java/org/elasticsearch/common/xcontent/XContentType.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index 43d36a800d345..a046adebf2ea4 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -169,7 +169,7 @@ public static String parseMediaType(String mediaType) {
         if (mediaType != null) {
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
             if (matcher.find()) {
-                return "application/" + matcher.group(2).toLowerCase();
+                return "application/" + matcher.group(2).toLowerCase(Locale.ROOT);
             }
         }
 

From 4eff534736a890668572710498c97741d985463a Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 17 Mar 2020 13:47:34 +0100
Subject: [PATCH 21/51] spotless

---
 .../rest/compat/version7/RestGetActionV7.java |  4 +---
 .../compat/version7/RestIndexActionV7.java    | 20 +++++++---------
 .../action/DocWriteResponseV7Tests.java       | 24 +++++++++----------
 .../compat/version7/RestGetActionV7Tests.java | 12 +++++-----
 .../version7/RestIndexActionV7Tests.java      | 10 +++-----
 5 files changed, 30 insertions(+), 40 deletions(-)

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index e00a600d073df..e9ac04ef4afbd 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -42,9 +42,7 @@ public class RestGetActionV7 extends RestGetAction {
     public List<Route> routes() {
         assert Version.CURRENT.major == 8 : "REST API compatibility for version 7 is only supported on version 8";
 
-        return List.of(
-            new Route(GET, "/{index}/{type}/{id}"),
-            new Route(HEAD, "/{index}/{type}/{id}"));
+        return List.of(new Route(GET, "/{index}/{type}/{id}"), new Route(HEAD, "/{index}/{type}/{id}"));
     }
 
     @Override
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
index 62af68b424342..1ce13121a4441 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -38,11 +38,10 @@
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestIndexActionV7 {
-    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " +
-        "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " +
-        "or /{index}/_create/{id}).";
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
-        LogManager.getLogger(RestIndexAction.class));
+    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document "
+        + "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, "
+        + "or /{index}/_create/{id}).";
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestIndexAction.class));
 
     private static void logDeprecationMessage() {
         deprecationLogger.deprecatedAndMaybeLog("index_with_types", TYPES_DEPRECATION_MESSAGE);
@@ -53,9 +52,7 @@ public static class CompatibleRestIndexAction extends RestIndexAction {
         public List<Route> routes() {
             assert Version.CURRENT.major == 8 : "REST API compatilbity for version 7 is only supported on version 8";
 
-            return List.of(
-                new Route(POST, "/{index}/{type}/{id}"),
-                new Route(PUT, "/{index}/{type}/{id}"));
+            return List.of(new Route(POST, "/{index}/{type}/{id}"), new Route(PUT, "/{index}/{type}/{id}"));
         }
 
         @Override
@@ -65,7 +62,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient
             return super.prepareRequest(request, client);
         }
 
-
         @Override
         public boolean compatibilityRequired() {
             return true;
@@ -75,9 +71,9 @@ public boolean compatibilityRequired() {
     public static class CompatibleCreateHandler extends RestIndexAction.CreateHandler {
         @Override
         public List<Route> routes() {
-            return unmodifiableList(asList(
-                new Route(POST, "/{index}/{type}/{id}/_create"),
-                new Route(PUT, "/{index}/{type}/{id}/_create")));
+            return unmodifiableList(
+                asList(new Route(POST, "/{index}/{type}/{id}/_create"), new Route(PUT, "/{index}/{type}/{id}/_create"))
+            );
         }
 
         @Override
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
index 0f3fcd7e85289..5815f08f6ee2f 100644
--- a/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/action/DocWriteResponseV7Tests.java
@@ -38,18 +38,18 @@
 public class DocWriteResponseV7Tests extends ESTestCase {
 
     public void testTypeWhenCompatible() throws IOException {
-        DocWriteResponse response =
-            new DocWriteResponse(
-                new ShardId("index", "uuid", 0),
-                "id",
-                SequenceNumbers.UNASSIGNED_SEQ_NO,
-                17,
-                0,
-                DocWriteResponse.Result.CREATED) {
-                // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
-            };
+        DocWriteResponse response = new DocWriteResponse(
+            new ShardId("index", "uuid", 0),
+            "id",
+            SequenceNumbers.UNASSIGNED_SEQ_NO,
+            17,
+            0,
+            DocWriteResponse.Result.CREATED
+        ) {
+            // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
+        };
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.setCompatibleMajorVersion((byte)7);
+            builder.setCompatibleMajorVersion((byte) 7);
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
 
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
@@ -58,7 +58,7 @@ public void testTypeWhenCompatible() throws IOException {
         }
 
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.setCompatibleMajorVersion((byte)6);
+            builder.setCompatibleMajorVersion((byte) 6);
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
 
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Tests.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Tests.java
index 31bb32d93ac25..c84972ad4bb33 100644
--- a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Tests.java
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestGetActionV7Tests.java
@@ -38,17 +38,17 @@ public void setUpAction() {
     }
 
     public void testTypeInPathWithGet() {
-        FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
-            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
-            .withPath("/some_index/some_type/some_id");
+        FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(
+            Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)
+        ).withPath("/some_index/some_type/some_id");
         dispatchRequest(deprecatedRequest.withMethod(RestRequest.Method.GET).build());
         assertWarnings(RestGetActionV7.TYPES_DEPRECATION_MESSAGE);
     }
 
     public void testTypeInPathWithHead() {
-        FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
-            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
-            .withPath("/some_index/some_type/some_id");
+        FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(
+            Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)
+        ).withPath("/some_index/some_type/some_id");
         dispatchRequest(deprecatedRequest.withMethod(RestRequest.Method.HEAD).build());
         assertWarnings(RestGetActionV7.TYPES_DEPRECATION_MESSAGE);
     }
diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Tests.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Tests.java
index d079f8d9600b3..59946abdd3d8e 100644
--- a/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Tests.java
+++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7Tests.java
@@ -35,7 +35,6 @@ public class RestIndexActionV7Tests extends RestActionTestCase {
     final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with=7");
     final List<String> contentTypeHeader = Collections.singletonList(mimeType);
 
-
     private final AtomicReference<ClusterState> clusterStateSupplier = new AtomicReference<>();
 
     @Before
@@ -47,8 +46,7 @@ public void setUpAction() {
 
     public void testTypeInPath() {
         // using CompatibleRestIndexAction
-        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
-            .withMethod(RestRequest.Method.PUT)
+        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT)
             .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
             .withPath("/some_index/some_type/some_id")
             .build();
@@ -58,8 +56,7 @@ public void testTypeInPath() {
 
     public void testCreateWithTypeInPath() {
         // using CompatibleCreateHandler
-        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
-            .withMethod(RestRequest.Method.PUT)
+        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT)
             .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
             .withPath("/some_index/some_type/some_id/_create")
             .build();
@@ -69,8 +66,7 @@ public void testCreateWithTypeInPath() {
 
     public void testAutoIdWithType() {
         // using CompatibleAutoIdHandler
-        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
-            .withMethod(RestRequest.Method.POST)
+        RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST)
             .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
             .withPath("/some_index/some_type/")
             .build();

From 2d4161cb7bdc9f52e49636a246f742c1a4ab0894 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 17 Mar 2020 14:23:27 +0100
Subject: [PATCH 22/51] imports and disable integ tests as there are none

---
 modules/rest-compatibility/build.gradle                       | 2 +-
 .../main/java/org/elasticsearch/rest/CompatibleConstants.java | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/modules/rest-compatibility/build.gradle b/modules/rest-compatibility/build.gradle
index ca0401d69fff3..96e81f07d823a 100644
--- a/modules/rest-compatibility/build.gradle
+++ b/modules/rest-compatibility/build.gradle
@@ -22,5 +22,5 @@ esplugin {
   classname 'org.elasticsearch.rest.compat.RestCompatPlugin'
 }
 
-integTest.enabled = true
+integTest.enabled = false
 test.enabled = true
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
index a86438695039f..78b62dac49ba7 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
@@ -20,10 +20,6 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.xcontent.ToXContent;
-
-import java.util.function.Consumer;
 
 public class CompatibleConstants {
 

From 6830f97d740edb9d35b348e9f1e06e538b3584fd Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 17 Mar 2020 15:12:07 +0100
Subject: [PATCH 23/51] fix tests

---
 .../org/elasticsearch/common/xcontent/XContentType.java     | 2 +-
 .../elasticsearch/common/xcontent/XContentTypeTests.java    | 2 +-
 .../org/elasticsearch/http/DefaultRestChannelTests.java     | 5 +++--
 .../java/org/elasticsearch/rest/RestControllerTests.java    | 6 ++++--
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index a046adebf2ea4..25c390c9a1142 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -179,7 +179,7 @@ public static String parseMediaType(String mediaType) {
     public static String parseVersion(String mediaType){
         if(mediaType != null){
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
-            if (matcher.find() && "vnd.elasticsearch+".equals(matcher.group(1))) {
+            if (matcher.find() && "vnd.elasticsearch+".equalsIgnoreCase(matcher.group(1))) {
 
                 return matcher.group(4);
             }
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
index c79beedc7ca18..856e544a23055 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
@@ -105,7 +105,7 @@ public void testMediaType() {
 
 
     public void testVersionParsing() {
-        byte version = randomByte();
+        String version = String.valueOf(Math.abs(randomByte()));
         assertThat(XContentType.parseVersion("application/vnd.elasticsearch+json;compatible-with=" + version),
             equalTo(version));
         assertThat(XContentType.parseVersion("application/vnd.elasticsearch+cbor;compatible-with=" + version),
diff --git a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
index 8c5beabfe0016..8d7a026aee034 100644
--- a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
+++ b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http;
 
 import org.apache.http.HttpHeaders;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -215,12 +216,12 @@ public void testHeadersSet() {
     }
 
     public void testCompatibleParamIsSet() {
-        final byte version = randomByte();
+        String version = String.valueOf(Version.CURRENT.major-1);
         final TestRequest httpRequest = new TestRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/");
         httpRequest.getHeaders().put(HttpHeaders.ACCEPT, List.of("application/vnd.elasticsearch+json;compatible-with=" + version));
         final RestRequest request = RestRequest.request(xContentRegistry(), httpRequest, httpChannel);
 
-        assertEquals("" + version, request.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY));
+        assertEquals(version, request.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY));
     }
 
     public void testCookiesSet() {
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index b8d4a63977bbf..0d51a0fcdbaeb 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -624,7 +625,9 @@ public void testDispatchRestrictSystemIndices() {
     }
 
     public void testDispatchCompatibleHandler() {
-        final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with=7");
+        final byte version = (byte) (Version.CURRENT.major - 1);
+
+        final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with="+version);
         String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
         final List<String> contentTypeHeader = Collections.singletonList(mimeType);
         FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
@@ -632,7 +635,6 @@ public void testDispatchCompatibleHandler() {
             .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
             .build();
         AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
-        final byte version = randomByte();
         restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
             @Override
             public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {

From 587f84d5711aa153f8b2cbcba7db2ab4a9fbb170 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 24 Mar 2020 16:28:51 +0100
Subject: [PATCH 24/51] Use content-type header with compatible api

When a request to a compatible api contains a body it will require a
content-type header to also contain a compatible media-type - in a form
application/vnd.elasticsearch+json;compatible-with=7
When there is no body - it will not be required.
Accept header is always required for compatible API
---
 .../rest/compat/AbstractCompatRestTest.java   | 16 +++---
 .../rest/CompatibleConstants.java             |  3 +-
 .../org/elasticsearch/rest/RestRequest.java   |  8 ++-
 .../rest/RestControllerTests.java             | 36 +++++++++++++
 .../elasticsearch/rest/RestRequestTests.java  | 53 +++++++++++++++++++
 5 files changed, 107 insertions(+), 9 deletions(-)

diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index 9cf2a780471fc..9f7f8bb91394d 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -77,16 +77,20 @@ private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
             //TODO: be more selective here
             doSection.setIgnoreWarnings(true);
 
-            String compatibleHeader = createCompatibleHeader();
-            //TODO decide which one to use - Accept or Content-Type
             doSection.getApiCallSection()
-                     .addHeaders(Map.of(
-                         CompatibleConstants.COMPATIBLE_HEADER, compatibleHeader,
-                         "Content-Type", compatibleHeader
-                         ));
+                     .addHeaders(createCompatibleHeaders(doSection));
         });
     }
 
+    private static Map<String, String> createCompatibleHeaders(DoSection doSection) {
+        String compatibleHeader = createCompatibleHeader();
+        if (doSection.getApiCallSection().hasBody()) {
+            return Map.of(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader,
+                CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader);
+        }
+        return Map.of(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader);
+    }
+
     private static String createCompatibleHeader() {
         return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleConstants.COMPATIBLE_VERSION;
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
index 78b62dac49ba7..e8effbe2c3448 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
@@ -26,7 +26,8 @@ public class CompatibleConstants {
     /**
      * TODO revisit when https://github.com/elastic/elasticsearch/issues/52370 is resolved
      */
-    public static final String COMPATIBLE_HEADER = "Accept";
+    public static final String COMPATIBLE_ACCEPT_HEADER = "Accept";
+    public static final String COMPATIBLE_CONTENT_TYPE_HEADER = "Content-Type";
     public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
     public static final String COMPATIBLE_VERSION = "" + (Version.CURRENT.major - 1);
 
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 63ebe4bf6e185..93f11ff8c55ea 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -139,7 +139,7 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
 
     private void addCompatibleParameter() {
         if (isRequestCompatible()) {
-            String compatibleVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_HEADER));
+            String compatibleVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
             params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, compatibleVersion);
             //use it so it won't fail request validation with unused parameter
             param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
@@ -147,7 +147,11 @@ private void addCompatibleParameter() {
     }
 
     private boolean isRequestCompatible() {
-        return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_HEADER));
+        if (hasContent()) {
+            return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER)) &&
+                isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER));
+        }
+        return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
     }
 
     private boolean isHeaderCompatible(String headerValue) {
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 0d51a0fcdbaeb..fd9adda9fab4d 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.usage.UsageService;
+import org.junit.Assert;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -659,6 +660,41 @@ public boolean compatibilityRequired() {
         assertTrue(channel.getSendResponseCalled());
     }
 
+    public void testIncorrectCompatibleHandlersDoNotDispatch() {
+        final byte version = (byte) (Version.CURRENT.major - 1);
+
+        final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with="+version);
+        String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
+        final List<String> contentTypeHeader = Collections.singletonList(mimeType);
+
+        // request to compatible api with body requires a content-type header.
+        // this one does no have it, making it non compatiblegit
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), RestRequest.parseContentType(contentTypeHeader)).withPath("/foo")
+            .withHeaders(Map.of("Accept", contentTypeHeader))
+            .build();
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                Assert.fail();
+            }
+
+            @Override
+            public boolean supportsContentStream() {
+                return true;
+            }
+
+            @Override
+            public boolean compatibilityRequired() {
+                return true;
+            }
+        });
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, false, RestStatus.BAD_REQUEST);
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+    }
+
     private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
         HttpServerTransport {
 
diff --git a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
index 487bbed5a5999..b46a07c4b12d5 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
+import java.net.http.HttpHeaders;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -41,8 +42,10 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -245,6 +248,56 @@ public void testRequiredContent() {
         assertEquals("unknown content type", e.getMessage());
     }
 
+    public void testCorrectCompatibleRequest() {
+        List<String> compatibleHeaderValue = List.of("application/vnd.elasticsearch+json;compatible-with=7");
+
+        // when no body - only Accept header is required
+        FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        builder.withHeaders(Map.of("Accept", compatibleHeaderValue));
+        RestRequest restRequest = builder.build();
+
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
+
+        // with body - both Accept and Content-Type are required
+        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        builder.withHeaders(Map.of("Accept", compatibleHeaderValue, "Content-Type", compatibleHeaderValue));
+        builder.withContent(new BytesArray("some content"), null);
+        restRequest = builder.build();
+
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
+    }
+
+    public void testIncorrectCompatibleRequest() {
+        List<String> compatibleHeaderValue = List.of("application/vnd.elasticsearch+json;compatible-with=7");
+
+        // no body, but no Accept header - won't set compatible param
+        FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        RestRequest restRequest = builder.build();
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
+
+        // no body, but incorrect header value
+        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        builder.withHeaders(Map.of("Accept", List.of("application/json")));
+        restRequest = builder.build();
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
+
+        // body provided, but no Content-Type
+        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        builder.withHeaders(Map.of("Accept", compatibleHeaderValue));
+        builder.withContent(new BytesArray("some content"), null);
+        restRequest = builder.build();
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
+
+
+        // body provided, but incorrect Content-Type
+        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        builder.withHeaders(Map.of("Accept", compatibleHeaderValue, "Content-Type", List.of("application/json")));
+        builder.withContent(new BytesArray("some content"), null);
+        restRequest = builder.build();
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
+
+    }
+
     private static RestRequest contentRestRequest(String content, Map<String, String> params) {
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("Content-Type", Collections.singletonList("application/json"));

From 7b092f5f1026aac195d5b185672183ac6856f6e8 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 24 Mar 2020 17:47:10 +0100
Subject: [PATCH 25/51] imports

---
 .../src/test/java/org/elasticsearch/rest/RestRequestTests.java  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
index b46a07c4b12d5..0799070f4ab64 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
-import java.net.http.HttpHeaders;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -42,7 +41,6 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;

From 2374c20ceb71e2ab5f8fe977579b31cf1a7884c0 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 1 Apr 2020 14:12:27 +0200
Subject: [PATCH 26/51] spotless

---
 .../rest/compat/AbstractCompatRestTest.java   | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index 9f7f8bb91394d..ea5a479dc4ff6 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -43,7 +43,6 @@ protected AbstractCompatRestTest(@Name("yaml") ClientYamlTestCandidate testCandi
         super(testCandidate);
     }
 
-
     private static final Logger staticLogger = LogManager.getLogger(AbstractCompatRestTest.class);
 
     public static final String COMPAT_TESTS_PATH = "/rest-api-spec/test-compat";
@@ -66,27 +65,29 @@ public static Iterable<Object[]> createParameters() throws Exception {
             });
             finalTestCandidates.add(testCandidates.toArray());
         }
-        localCandidates.keySet().forEach(lc -> finalTestCandidates.add(new Object[]{lc}));
+        localCandidates.keySet().forEach(lc -> finalTestCandidates.add(new Object[] { lc }));
         return finalTestCandidates;
     }
 
-
     private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
         testCandidate.getTestSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(ds -> {
             DoSection doSection = (DoSection) ds;
-            //TODO: be more selective here
+            // TODO: be more selective here
             doSection.setIgnoreWarnings(true);
 
-            doSection.getApiCallSection()
-                     .addHeaders(createCompatibleHeaders(doSection));
+            doSection.getApiCallSection().addHeaders(createCompatibleHeaders(doSection));
         });
     }
 
     private static Map<String, String> createCompatibleHeaders(DoSection doSection) {
         String compatibleHeader = createCompatibleHeader();
         if (doSection.getApiCallSection().hasBody()) {
-            return Map.of(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader,
-                CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader);
+            return Map.of(
+                CompatibleConstants.COMPATIBLE_ACCEPT_HEADER,
+                compatibleHeader,
+                CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER,
+                compatibleHeader
+            );
         }
         return Map.of(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader);
     }
@@ -95,13 +96,12 @@ private static String createCompatibleHeader() {
         return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleConstants.COMPATIBLE_VERSION;
     }
 
-
     private static Map<ClientYamlTestCandidate, ClientYamlTestCandidate> getLocalCompatibilityTests() throws Exception {
-        Iterable<Object[]> candidates =
-            ESClientYamlSuiteTestCase.createParameters(ExecutableSection.XCONTENT_REGISTRY, COMPAT_TESTS_PATH);
+        Iterable<Object[]> candidates = ESClientYamlSuiteTestCase.createParameters(ExecutableSection.XCONTENT_REGISTRY, COMPAT_TESTS_PATH);
         Map<ClientYamlTestCandidate, ClientYamlTestCandidate> localCompatibilityTests = new HashMap<>();
         StreamSupport.stream(candidates.spliterator(), false)
-            .flatMap(Arrays::stream).forEach(o -> localCompatibilityTests.put((ClientYamlTestCandidate) o, (ClientYamlTestCandidate) o));
+            .flatMap(Arrays::stream)
+            .forEach(o -> localCompatibilityTests.put((ClientYamlTestCandidate) o, (ClientYamlTestCandidate) o));
         return localCompatibilityTests;
     }
 }

From eeae41f4bf6835d23dde34f706442824e5f98766 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 1 Apr 2020 14:41:40 +0200
Subject: [PATCH 27/51] fix compile test

---
 .../rest/action/document/RestIndexActionTests.java            | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
index 84bc5f3bef340..08cf6fd656d44 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
@@ -53,11 +53,9 @@ public class RestIndexActionTests extends RestActionTestCase {
 
     @Before
     public void setUpAction() {
-        ClusterService clusterService = mock(ClusterService.class);
-        when(clusterService.state()).thenAnswer(invocationOnMock -> clusterStateSupplier.get());
         controller().registerHandler(new RestIndexAction());
         controller().registerHandler(new CreateHandler());
-        controller().registerHandler(new AutoIdHandler(clusterService));
+        controller().registerHandler(new AutoIdHandler(() -> clusterStateSupplier.get().nodes()));
     }
 
     public void testCreateOpTypeValidation() {

From e7d9b2906d23dbe5971f3a56ac95757411ad7beb Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 1 Apr 2020 14:49:53 +0200
Subject: [PATCH 28/51] checkstyle

---
 .../rest/action/document/RestIndexActionTests.java             | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
index 08cf6fd656d44..bb549b64c724f 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
@@ -43,9 +42,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
 public class RestIndexActionTests extends RestActionTestCase {
 

From 268bc1c2aedcaa439cccf38971850f04e4e212b5 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 2 Apr 2020 14:53:53 +0200
Subject: [PATCH 29/51] rename of mimetype header

---
 .../rest/RestControllerTests.java             | 39 +++++++++----------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 2b440ee024c60..dafbacef28f35 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -59,6 +59,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -609,12 +610,13 @@ public HttpRequest releaseAndCopy() {
     public void testDispatchCompatibleHandler() {
         final byte version = (byte) (Version.CURRENT.major - 1);
 
-        final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with="+version);
+        final String mimeType = randomCompatibleMimeType(version);
         String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
-        final List<String> contentTypeHeader = Collections.singletonList(mimeType);
+        final List<String> mimeTypeList = Collections.singletonList(mimeType);
         FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
-            .withContent(new BytesArray(content), RestRequest.parseContentType(contentTypeHeader)).withPath("/foo")
-            .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader))
+            .withContent(new BytesArray(content), RestRequest.parseContentType(mimeTypeList))
+            .withPath("/foo")
+            .withHeaders(Map.of("Content-Type", mimeTypeList, "Accept", mimeTypeList))
             .build();
         AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
         restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
@@ -625,11 +627,6 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
                 channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
             }
 
-            @Override
-            public boolean supportsContentStream() {
-                return true;
-            }
-
             @Override
             public boolean compatibilityRequired() {
                 return true;
@@ -641,18 +638,25 @@ public boolean compatibilityRequired() {
         assertTrue(channel.getSendResponseCalled());
     }
 
+    private String randomCompatibleMimeType(byte version) {
+        String subtype = randomFrom(Stream.of(XContentType.values())
+            .map(XContentType::shortName)
+            .toArray(String[]::new));
+        return randomFrom("application/vnd.elasticsearch+" + subtype + ";compatible-with=" + version);
+    }
+
     public void testIncorrectCompatibleHandlersDoNotDispatch() {
         final byte version = (byte) (Version.CURRENT.major - 1);
-
-        final String mimeType = randomFrom("application/vnd.elasticsearch+json;compatible-with="+version);
+        final String mimeType = randomCompatibleMimeType(version);
         String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
-        final List<String> contentTypeHeader = Collections.singletonList(mimeType);
+        final List<String> mimeTypeList = Collections.singletonList(mimeType);
 
         // request to compatible api with body requires a content-type header.
-        // this one does no have it, making it non compatiblegit
+        // this one does no have it, making it non compatible
         FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
-            .withContent(new BytesArray(content), RestRequest.parseContentType(contentTypeHeader)).withPath("/foo")
-            .withHeaders(Map.of("Accept", contentTypeHeader))
+            .withContent(new BytesArray(content), RestRequest.parseContentType(mimeTypeList))
+            .withPath("/foo")
+            .withHeaders(Map.of("Accept", mimeTypeList))
             .build();
         restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
             @Override
@@ -660,11 +664,6 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
                 Assert.fail();
             }
 
-            @Override
-            public boolean supportsContentStream() {
-                return true;
-            }
-
             @Override
             public boolean compatibilityRequired() {
                 return true;

From 15f4be51f6374b4b8088eb8811beed432110fa65 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 2 Apr 2020 15:07:33 +0200
Subject: [PATCH 30/51] additional header test

---
 .../java/org/elasticsearch/rest/RestRequestTests.java     | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
index 0799070f4ab64..5a5b2c924c65f 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
@@ -253,15 +253,21 @@ public void testCorrectCompatibleRequest() {
         FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
         builder.withHeaders(Map.of("Accept", compatibleHeaderValue));
         RestRequest restRequest = builder.build();
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
 
+        // when no body - both Accept and Content-Type provided, but only Accept required - still OK
+        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        builder.withHeaders(Map.of("Accept", compatibleHeaderValue, "Content-Type", compatibleHeaderValue));
+        builder.withContent(new BytesArray("some content"), null);
+        restRequest = builder.build();
         assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
 
+
         // with body - both Accept and Content-Type are required
         builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
         builder.withHeaders(Map.of("Accept", compatibleHeaderValue, "Content-Type", compatibleHeaderValue));
         builder.withContent(new BytesArray("some content"), null);
         restRequest = builder.build();
-
         assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
     }
 

From 3d975f38ae7e2c776e53cd282fb249cf678a1471 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 6 Apr 2020 12:00:25 +0200
Subject: [PATCH 31/51] initial headers validation

---
 .../rest/compat/version7/RestGetActionV7.java |   5 +
 .../compat/version7/RestIndexActionV7.java    |  16 +++
 .../http/AbstractHttpServerTransport.java     |   3 +
 .../org/elasticsearch/rest/RestRequest.java   |  20 ++-
 .../CompatibleHeaderCombinationTests.java     | 135 ++++++++++++++++++
 5 files changed, 177 insertions(+), 2 deletions(-)
 create mode 100644 server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index e9ac04ef4afbd..d8d01f42698f7 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -38,6 +38,11 @@ public class RestGetActionV7 extends RestGetAction {
     static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
         + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
 
+    @Override
+    public String getName() {
+        return "document_get_action_v7";
+    }
+
     @Override
     public List<Route> routes() {
         assert Version.CURRENT.major == 8 : "REST API compatibility for version 7 is only supported on version 8";
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
index 1ce13121a4441..51d4a56baa8fa 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -48,6 +48,11 @@ private static void logDeprecationMessage() {
     }
 
     public static class CompatibleRestIndexAction extends RestIndexAction {
+        @Override
+        public String getName() {
+            return "document_index_action_v7";
+        }
+
         @Override
         public List<Route> routes() {
             assert Version.CURRENT.major == 8 : "REST API compatilbity for version 7 is only supported on version 8";
@@ -69,6 +74,12 @@ public boolean compatibilityRequired() {
     }
 
     public static class CompatibleCreateHandler extends RestIndexAction.CreateHandler {
+
+        @Override
+        public String getName() {
+            return "document_create_action_v7";
+        }
+
         @Override
         public List<Route> routes() {
             return unmodifiableList(
@@ -95,6 +106,11 @@ public CompatibleAutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
             super(nodesInCluster);
         }
 
+        @Override
+        public String getName() {
+            return "document_create_action_auto_id_v7";
+        }
+
         @Override
         public List<Route> routes() {
             return singletonList(new Route(POST, "/{index}/{type}"));
diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
index 8d02eac5b8b13..dee9325f8827e 100644
--- a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
+++ b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
@@ -352,6 +352,9 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan
             } catch (final RestRequest.BadParameterException e) {
                 badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e);
                 innerRestRequest = RestRequest.requestWithoutParameters(xContentRegistry, httpRequest, httpChannel);
+            } catch (final RestRequest.CompatibleApiHeadersCombinationException e){
+                badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e);
+                innerRestRequest = requestWithoutContentTypeHeader(httpRequest, httpChannel, badRequestCause);
             }
             restRequest = innerRestRequest;
         }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 3c7a932eb578f..c227f2e131396 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -147,8 +147,16 @@ private void addCompatibleParameter() {
 
     private boolean isRequestCompatible() {
         if (hasContent()) {
-            return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER)) &&
-                isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER));
+            String acceptHeader = header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER);
+            String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
+            if (isHeaderCompatible(acceptHeader) && isHeaderCompatible(contentTypeHeader)) {
+                return true;
+            } else if (isHeaderCompatible(acceptHeader) && isHeaderCompatible(contentTypeHeader) == false) {
+                throw new CompatibleApiHeadersCombinationException(
+                    String.format("Request with a body and compatible Accept={} but incorrect Content-Type={}",
+                        acceptHeader, contentTypeHeader));
+            }
+            return false;
         }
         return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
     }
@@ -562,4 +570,12 @@ public static class BadParameterException extends RuntimeException {
 
     }
 
+    public static class CompatibleApiHeadersCombinationException extends RuntimeException {
+
+        CompatibleApiHeadersCombinationException(String cause) {
+            super(cause);
+        }
+
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
new file mode 100644
index 0000000000000..51cccec7d2359
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchMatchers;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.hamcrest.FeatureMatcher;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+import org.junit.rules.ExpectedException;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.hamcrest.Matchers.*;
+
+public class CompatibleHeaderCombinationTests extends ESTestCase {
+    public void testNotACompatibleApiRequest() {
+        // no body, and no Accept header - won't set compatible param. Not a compatible Request
+        FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        FakeRestRequest restRequest = builder.build();
+        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
+    }
+
+    public void testIncorrectCompatibleAcceptHeader() {
+        testRestRequestCreation(correctV7Accept(), correctV7ContentType(), bodyPresent(),
+            expect(requestCreated(), isCompatible()));
+
+        testRestRequestCreation(correctV7Accept(), correctV7ContentType(), bodyNotPresent(),
+            expect(requestCreated(), isCompatible()));
+
+        testRestRequestCreation(correctV7Accept(), v8ContentType(), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+    }
+
+    private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends Exception> exceptionClass) {
+        return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {
+            try {
+                builder.build();
+            } catch (Exception e) {
+                return e;
+            }
+            return null;
+        }, instanceOf(exceptionClass));
+
+    }
+
+    private Matcher<FakeRestRequest.Builder> requestCreated() {
+        return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> builder.build(), Matchers.notNullValue());
+    }
+
+    private Matcher<FakeRestRequest.Builder> isCompatible() {
+        return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {
+            FakeRestRequest build = builder.build();
+            return build.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY); //TODO to be refactored into getVersion
+        }, equalTo(String.valueOf(Version.CURRENT.major - 1)));
+    }
+
+    private String bodyNotPresent() {
+        return null;
+    }
+
+    private String bodyPresent() {
+        return "some body";
+    }
+
+    private List<String> correctV7ContentType() {
+        return mediaType(7);
+    }
+
+    private List<String> correctV7Accept() {
+        return mediaType(7);
+    }
+
+    private List<String> v8ContentType() {
+        return mediaType(8);
+    }
+
+    private List<String> mediaType(int version) {
+        return List.of("application/vnd.elasticsearch+json;compatible-with=" + version);
+    }
+
+    private Matcher<FakeRestRequest.Builder> expect(Matcher<FakeRestRequest.Builder>... matchers) {
+        return Matchers.allOf(matchers);
+    }
+
+    private void testRestRequestCreation(List<String> accept,
+                                         List<String> contentType,
+                                         String body,
+                                         Matcher<FakeRestRequest.Builder> matcher) {
+        FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+
+        builder.withHeaders(createHeaders(accept, contentType));
+        if (body != null) {
+            // xContentType header is set explicitly in headers
+            builder.withContent(new BytesArray(body), null);
+        }
+        assertThat(builder, matcher);
+    }
+
+    private Map<String, List<String>> createHeaders(List<String> accept, List<String> contentType) {
+        Map<String, List<String>> headers = new HashMap<>();
+        if (accept != null) {
+            headers.put("Accept", accept);
+        }
+        if (accept != null) {
+            headers.put("Content-Type", contentType);
+        }
+        return headers;
+    }
+}

From 09ef281814fc9e8ba5503e30a0807450a90c9504 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 6 Apr 2020 14:00:45 +0200
Subject: [PATCH 32/51] test cases

---
 .../org/elasticsearch/rest/RestRequest.java   | 36 +++++---
 .../CompatibleHeaderCombinationTests.java     | 89 ++++++++++++++-----
 2 files changed, 93 insertions(+), 32 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index c227f2e131396..329dfd5d8b587 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Nullable;
@@ -46,6 +47,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Pattern;
@@ -137,6 +139,7 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
     }
 
     private void addCompatibleParameter() {
+        validateParameters();
         if (isRequestCompatible()) {
             String compatibleVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
             params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, compatibleVersion);
@@ -145,18 +148,31 @@ private void addCompatibleParameter() {
         }
     }
 
-    private boolean isRequestCompatible() {
+    private void validateParameters() {
+        String currentVersion = String.valueOf(Version.CURRENT.major);
+        String previousVersion = String.valueOf(Version.CURRENT.major - 1);
+        String acceptVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
+        String contentTypeVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER));
+
         if (hasContent()) {
-            String acceptHeader = header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER);
-            String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
-            if (isHeaderCompatible(acceptHeader) && isHeaderCompatible(contentTypeHeader)) {
-                return true;
-            } else if (isHeaderCompatible(acceptHeader) && isHeaderCompatible(contentTypeHeader) == false) {
-                throw new CompatibleApiHeadersCombinationException(
-                    String.format("Request with a body and compatible Accept={} but incorrect Content-Type={}",
-                        acceptHeader, contentTypeHeader));
+            if ((previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion)) &&
+                acceptVersion.equals(contentTypeVersion)) {
+                return;
             }
-            return false;
+
+        } else if (previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion)) {
+            return;
+        }
+        throw new CompatibleApiHeadersCombinationException(
+            String.format("Request with a body and compatible Accept=%s but incorrect Content-Type=%s",
+                String.valueOf(acceptVersion), String.valueOf(contentTypeVersion)));
+
+    }
+
+    private boolean isRequestCompatible() {
+        if (hasContent()) {
+            return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER)) &&
+                isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER));
         }
         return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
     }
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 51cccec7d2359..dd3070ea170a4 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -25,20 +25,19 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchMatchers;
 import org.elasticsearch.test.rest.FakeRestRequest;
-import org.hamcrest.FeatureMatcher;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
-import org.junit.rules.ExpectedException;
 
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.*;
 
 public class CompatibleHeaderCombinationTests extends ESTestCase {
+    int CURRENT_VERSION = Version.CURRENT.major;
+    int PREVIOUS_VERSION = Version.CURRENT.major - 1;
+
     public void testNotACompatibleApiRequest() {
         // no body, and no Accept header - won't set compatible param. Not a compatible Request
         FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
@@ -47,21 +46,64 @@ public void testNotACompatibleApiRequest() {
     }
 
     public void testIncorrectCompatibleAcceptHeader() {
-        testRestRequestCreation(correctV7Accept(), correctV7ContentType(), bodyPresent(),
+        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(requestCreated(), isCompatible()));
 
-        testRestRequestCreation(correctV7Accept(), correctV7ContentType(), bodyNotPresent(),
+        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
             expect(requestCreated(), isCompatible()));
 
-        testRestRequestCreation(correctV7Accept(), v8ContentType(), bodyPresent(),
+        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
+            expect(requestCreated(), isCompatible())); // no body - content-type is ignored
+
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible()))); // no body - content-type is ignored
+
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+        //tests when body present and one of the headers missing
+        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        //tests when body NOT present and one of the headers missing
+        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent(),
+            expect(requestCreated(), isCompatible()));
+
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+// test cases from the doc, but not sure why would these be ok when accept is null
+//        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
+//            expect(requestCreated(), isCompatible()));
+//        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
+//            expect(requestCreated(), isCompatible()));
     }
 
+
     private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends Exception> exceptionClass) {
         return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {
             try {
                 builder.build();
             } catch (Exception e) {
+                e.printStackTrace();
                 return e;
             }
             return null;
@@ -74,10 +116,14 @@ private Matcher<FakeRestRequest.Builder> requestCreated() {
     }
 
     private Matcher<FakeRestRequest.Builder> isCompatible() {
+        return requestHasVersion(Version.CURRENT.major - 1);
+    }
+
+    private Matcher<FakeRestRequest.Builder> requestHasVersion(int version) {
         return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {
             FakeRestRequest build = builder.build();
             return build.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY); //TODO to be refactored into getVersion
-        }, equalTo(String.valueOf(Version.CURRENT.major - 1)));
+        }, equalTo(String.valueOf(version)));
     }
 
     private String bodyNotPresent() {
@@ -88,30 +134,29 @@ private String bodyPresent() {
         return "some body";
     }
 
-    private List<String> correctV7ContentType() {
-        return mediaType(7);
-    }
-
-    private List<String> correctV7Accept() {
-        return mediaType(7);
+    private List<String> contentTypeHeader(Integer version) {
+        return mediaType(version);
     }
 
-    private List<String> v8ContentType() {
-        return mediaType(8);
+    private List<String> acceptHeader(Integer version) {
+        return mediaType(version);
     }
 
-    private List<String> mediaType(int version) {
-        return List.of("application/vnd.elasticsearch+json;compatible-with=" + version);
+    private List<String> mediaType(Integer version) {
+        if (version != null) {
+            return List.of("application/vnd.elasticsearch+json;compatible-with=" + version);
+        }
+        return null;
     }
 
     private Matcher<FakeRestRequest.Builder> expect(Matcher<FakeRestRequest.Builder>... matchers) {
         return Matchers.allOf(matchers);
     }
 
-    private void testRestRequestCreation(List<String> accept,
-                                         List<String> contentType,
-                                         String body,
-                                         Matcher<FakeRestRequest.Builder> matcher) {
+    private void createRequestWith(List<String> accept,
+                                   List<String> contentType,
+                                   String body,
+                                   Matcher<FakeRestRequest.Builder> matcher) {
         FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
 
         builder.withHeaders(createHeaders(accept, contentType));

From bf09243419228b9a111b650fdcbf3fde699243fb Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 6 Apr 2020 14:02:00 +0200
Subject: [PATCH 33/51] remove unused testcase

---
 .../rest/CompatibleHeaderCombinationTests.java           | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index dd3070ea170a4..1158207cda8b0 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -38,14 +38,7 @@ public class CompatibleHeaderCombinationTests extends ESTestCase {
     int CURRENT_VERSION = Version.CURRENT.major;
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
 
-    public void testNotACompatibleApiRequest() {
-        // no body, and no Accept header - won't set compatible param. Not a compatible Request
-        FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        FakeRestRequest restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
-    }
-
-    public void testIncorrectCompatibleAcceptHeader() {
+    public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(requestCreated(), isCompatible()));
 

From 30f1c40432695232e5f8c4dae11c671cdc005e4b Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Mon, 6 Apr 2020 14:52:34 +0200
Subject: [PATCH 34/51] precommit

---
 .../src/main/java/org/elasticsearch/rest/RestRequest.java  | 7 +++----
 .../rest/CompatibleHeaderCombinationTests.java             | 5 +++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 329dfd5d8b587..86cd9bab64b41 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -46,8 +46,8 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Pattern;
@@ -164,9 +164,8 @@ private void validateParameters() {
             return;
         }
         throw new CompatibleApiHeadersCombinationException(
-            String.format("Request with a body and compatible Accept=%s but incorrect Content-Type=%s",
-                String.valueOf(acceptVersion), String.valueOf(contentTypeVersion)));
-
+            String.format(Locale.ROOT, "Request with a body and compatible Accept=%s but incorrect Content-Type=%s",
+                acceptVersion, contentTypeVersion));
     }
 
     private boolean isRequestCompatible() {
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 1158207cda8b0..97c8efe8fc940 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -32,7 +32,9 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.*;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 
 public class CompatibleHeaderCombinationTests extends ESTestCase {
     int CURRENT_VERSION = Version.CURRENT.major;
@@ -96,7 +98,6 @@ private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends
             try {
                 builder.build();
             } catch (Exception e) {
-                e.printStackTrace();
                 return e;
             }
             return null;

From a013a0336f88f71837fe282ec488e97e78065dc6 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 7 Apr 2020 12:26:52 +0200
Subject: [PATCH 35/51] headers validation

---
 .../common/xcontent/XContentType.java         |  7 ++-
 .../org/elasticsearch/rest/RestRequest.java   | 29 ++++++++----
 .../CompatibleHeaderCombinationTests.java     | 46 ++++++++++++++++---
 3 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index 25c390c9a1142..5378c3c46eb79 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -126,7 +126,9 @@ public XContent xContent() {
      * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a
      * format query string parameter. This method will return {@code null} if no match is found
      */
-    public static XContentType fromMediaTypeOrFormat(String mediaType) {
+    public static XContentType fromMediaTypeOrFormat(String mediaTypeHeaderValue) {
+        String mediaType = parseMediaType(mediaTypeHeaderValue);
+
         if (mediaType == null) {
             return null;
         }
@@ -165,7 +167,8 @@ public static XContentType fromMediaType(String mediaTypeHeaderValue) {
         return null;
     }
 
-    public static String parseMediaType(String mediaType) {
+    //default scope for testing
+    static String parseMediaType(String mediaType) {
         if (mediaType != null) {
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
             if (matcher.find()) {
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 86cd9bab64b41..090fc6561cc6a 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -139,7 +139,7 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
     }
 
     private void addCompatibleParameter() {
-        validateParameters();
+        validateAcceptContentTypeHeaders();
         if (isRequestCompatible()) {
             String compatibleVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
             params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, compatibleVersion);
@@ -148,24 +148,37 @@ private void addCompatibleParameter() {
         }
     }
 
-    private void validateParameters() {
+    private void validateAcceptContentTypeHeaders() {
         String currentVersion = String.valueOf(Version.CURRENT.major);
         String previousVersion = String.valueOf(Version.CURRENT.major - 1);
-        String acceptVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
-        String contentTypeVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER));
+        String acceptVersion = getVersion(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER);
+        String contentTypeVersion = getVersion(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
 
         if (hasContent()) {
-            if ((previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion)) &&
+            if ((previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion) ) &&
                 acceptVersion.equals(contentTypeVersion)) {
                 return;
             }
 
-        } else if (previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion)) {
+        } else if ( previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion) || acceptVersion == null) {
             return;
         }
         throw new CompatibleApiHeadersCombinationException(
-            String.format(Locale.ROOT, "Request with a body and compatible Accept=%s but incorrect Content-Type=%s",
-                acceptVersion, contentTypeVersion));
+            String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
+                    "Accept=%s Content-Type=%s", acceptVersion, contentTypeVersion));
+    }
+
+    private String getVersion(String headerName) {
+        String headerValue = header(headerName);
+        String version = XContentType.parseVersion(headerValue);
+        if(version != null){
+            return version;
+        }
+        // if version was not provided but mediaType is parsable - return Current
+        if(XContentType.fromMediaTypeOrFormat(headerValue) != null){
+            return String.valueOf(Version.CURRENT.major);
+        }
+        return headerValue;
     }
 
     private boolean isRequestCompatible() {
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 97c8efe8fc940..ce8fd3c3bb7a3 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -85,13 +85,30 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
 
-// test cases from the doc, but not sure why would these be ok when accept is null
-//        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-//            expect(requestCreated(), isCompatible()));
-//        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
-//            expect(requestCreated(), isCompatible()));
+        //body not present - ignoring content-type. Accept header = null defaults to JSON and current version (non compatible)
+        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader(null), contentTypeHeader(null), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+        //Accept header = application/json means current version. If body is provided then accept and content-Type should be the same
+        createRequestWith(acceptHeader("application/json"), contentTypeHeader(null), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/json"), bodyPresent(),
+            expect(requestCreated(), not(isCompatible())));
     }
 
+    public void testMediaTypeCombinations(){
+        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/smile"), bodyPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+    }
 
     private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends Exception> exceptionClass) {
         return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {
@@ -128,14 +145,29 @@ private String bodyPresent() {
         return "some body";
     }
 
-    private List<String> contentTypeHeader(Integer version) {
+    private List<String> contentTypeHeader(int version) {
         return mediaType(version);
     }
 
-    private List<String> acceptHeader(Integer version) {
+    private List<String> acceptHeader(int version) {
         return mediaType(version);
     }
 
+    private List<String> acceptHeader(String value) {
+        return headerValue(value);
+    }
+
+    private List<String> contentTypeHeader(String value) {
+        return headerValue(value);
+    }
+
+    private List<String> headerValue(String value) {
+        if (value != null) {
+            return List.of(value);
+        }
+        return null;
+    }
+
     private List<String> mediaType(Integer version) {
         if (version != null) {
             return List.of("application/vnd.elasticsearch+json;compatible-with=" + version);

From 6c284342a2195a57d43b3a2f04678f5634212c91 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 8 Apr 2020 15:08:21 +0200
Subject: [PATCH 36/51] 273 failing

---
 .../gradle/http/WaitForHttpResource.java      |  3 ++
 .../rest/compat/AbstractCompatRestTest.java   | 34 +++++++++----------
 .../rest/CompatibleConstants.java             |  2 +-
 .../org/elasticsearch/rest/RestRequest.java   |  7 +++-
 .../CompatibleHeaderCombinationTests.java     | 19 +++++++----
 .../test/rest/yaml/ClientYamlTestClient.java  |  4 +++
 .../yaml/ClientYamlTestExecutionContext.java  | 17 ++++++++--
 7 files changed, 56 insertions(+), 30 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java b/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
index db805b26cdca5..d097fabbbbced 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
@@ -130,6 +130,8 @@ public boolean wait(int durationInMs) throws GeneralSecurityException, Interrupt
 
     protected void checkResource(SSLContext ssl) throws IOException {
         final HttpURLConnection connection = buildConnection(ssl);
+        connection.setRequestProperty("Accept", "application/json");
+
         connection.connect();
         final Integer response = connection.getResponseCode();
         if (validResponseCodes.contains(response)) {
@@ -144,6 +146,7 @@ HttpURLConnection buildConnection(SSLContext ssl) throws IOException {
         final HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
         configureSslContext(connection, ssl);
         configureBasicAuth(connection);
+
         connection.setRequestMethod("GET");
         return connection;
     }
diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index ea5a479dc4ff6..5b3dd3a7e052d 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -33,6 +33,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 import java.util.stream.StreamSupport;
 
 /**
@@ -70,27 +71,24 @@ public static Iterable<Object[]> createParameters() throws Exception {
     }
 
     private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
-        testCandidate.getTestSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(ds -> {
+        testCandidate.getSetupSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(updateDoSection());
+        testCandidate.getTestSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(updateDoSection());
+    }
+
+    private static Consumer<? super ExecutableSection> updateDoSection() {
+        return ds -> {
             DoSection doSection = (DoSection) ds;
-            // TODO: be more selective here
+            //TODO: be more selective here
             doSection.setIgnoreWarnings(true);
 
-            doSection.getApiCallSection().addHeaders(createCompatibleHeaders(doSection));
-        });
-    }
-
-    private static Map<String, String> createCompatibleHeaders(DoSection doSection) {
-        String compatibleHeader = createCompatibleHeader();
-        if (doSection.getApiCallSection().hasBody()) {
-            return Map.of(
-                CompatibleConstants.COMPATIBLE_ACCEPT_HEADER,
-                compatibleHeader,
-                CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER,
-                compatibleHeader
-            );
-        }
-        return Map.of(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader);
-    }
+            String compatibleHeader = createCompatibleHeader();
+            doSection.getApiCallSection()
+                .addHeaders(Map.of(
+                    CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader,
+                    CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
+                ));
+        };
+}
 
     private static String createCompatibleHeader() {
         return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleConstants.COMPATIBLE_VERSION;
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
index e8effbe2c3448..14c10345a23b6 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
@@ -29,6 +29,6 @@ public class CompatibleConstants {
     public static final String COMPATIBLE_ACCEPT_HEADER = "Accept";
     public static final String COMPATIBLE_CONTENT_TYPE_HEADER = "Content-Type";
     public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
-    public static final String COMPATIBLE_VERSION = "" + (Version.CURRENT.major - 1);
+    public static final String COMPATIBLE_VERSION = String.valueOf(Version.CURRENT.major - 1);
 
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 090fc6561cc6a..7b62f57d43d3f 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -159,13 +159,18 @@ private void validateAcceptContentTypeHeaders() {
                 acceptVersion.equals(contentTypeVersion)) {
                 return;
             }
+            if(XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER)) == null &&
+                XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER)) == null){
+                return;
+            }
 
         } else if ( previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion) || acceptVersion == null) {
             return;
         }
         throw new CompatibleApiHeadersCombinationException(
             String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                    "Accept=%s Content-Type=%s", acceptVersion, contentTypeVersion));
+                    "Accept=%s Content-Type=%s hasContent=%b %s %s %s", header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER),
+                header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER), hasContent(),path(), params.toString(),method().toString()));
     }
 
     private String getVersion(String headerName) {
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index ce8fd3c3bb7a3..238326dc57f02 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -40,6 +40,10 @@ public class CompatibleHeaderCombinationTests extends ESTestCase {
     int CURRENT_VERSION = Version.CURRENT.major;
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
 
+    public void testOld(){
+        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+    }
     public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(requestCreated(), isCompatible()));
@@ -101,13 +105,14 @@ public void testAcceptAndContentTypeCombinations() {
     }
 
     public void testMediaTypeCombinations(){
-        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/smile"), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
 
-        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+//        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/smile"), bodyPresent(),
+//            expect(requestCreated(), not(isCompatible())));
+
+//        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
+//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+//        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
+//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
     }
 
     private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends Exception> exceptionClass) {
@@ -198,7 +203,7 @@ private Map<String, List<String>> createHeaders(List<String> accept, List<String
         if (accept != null) {
             headers.put("Accept", accept);
         }
-        if (accept != null) {
+        if (contentType != null) {
             headers.put("Content-Type", contentType);
         }
         return headers;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
index 64b61773461f6..c0d589fb2d9a3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
@@ -23,6 +23,7 @@
 import org.apache.http.HttpHost;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.entity.ContentType;
+import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -179,6 +180,9 @@ public ClientYamlTestResponse callApi(String apiName, Map<String, String> params
         for (Map.Entry<String, String> param : queryStringParams.entrySet()) {
             request.addParameter(param.getKey(), param.getValue());
         }
+        if(entity!=null && entity.getContentType() != null){
+            headers.remove(HTTP.CONTENT_TYPE);
+        }
         request.setEntity(entity);
         setOptions(request, headers);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
index b1337172a5679..c48bcdd653a39 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
@@ -23,6 +23,7 @@
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
+import org.apache.http.protocol.HTTP;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
@@ -118,8 +119,13 @@ private HttpEntity createEntity(List<Map<String, Object>> bodies, Map<String, St
         if (bodies.size() == 1) {
             XContentType xContentType = getContentType(headers, XContentType.values());
             BytesRef bytesRef = bodyAsBytesRef(bodies.get(0), xContentType);
-            return new ByteArrayEntity(bytesRef.bytes, bytesRef.offset, bytesRef.length,
-                    ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
+            //TODO clean this up
+            ByteArrayEntity byteArrayEntity = new ByteArrayEntity(bytesRef.bytes, bytesRef.offset, bytesRef.length,
+                ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
+            if(byteArrayEntity.getContentType() != null){
+                byteArrayEntity.setContentType(headers.get(HTTP.CONTENT_TYPE));
+            }
+            return byteArrayEntity;
         } else {
             XContentType xContentType = getContentType(headers, STREAMING_CONTENT_TYPES);
             List<BytesRef> bytesRefList = new ArrayList<>(bodies.size());
@@ -137,7 +143,12 @@ private HttpEntity createEntity(List<Map<String, Object>> bodies, Map<String, St
                 }
                 bytes[position++] = xContentType.xContent().streamSeparator();
             }
-            return new ByteArrayEntity(bytes, ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
+            //TODO clean this up
+            ByteArrayEntity byteArrayEntity = new ByteArrayEntity(bytes, ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
+            if(byteArrayEntity.getContentType() != null){
+                byteArrayEntity.setContentType(headers.get(HTTP.CONTENT_TYPE));
+            }
+            return byteArrayEntity;
         }
     }
 

From 37d6426665b52cc2eaac65029ac35a3d0daea95c Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 8 Apr 2020 18:31:07 +0200
Subject: [PATCH 37/51] *.*

---
 .../rest/compat/AbstractCompatRestTest.java    | 18 +++++++++++++-----
 .../org/elasticsearch/rest/RestRequest.java    |  3 ++-
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index 5b3dd3a7e052d..c67901f3720fe 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -82,11 +82,19 @@ private static Consumer<? super ExecutableSection> updateDoSection() {
             doSection.setIgnoreWarnings(true);
 
             String compatibleHeader = createCompatibleHeader();
-            doSection.getApiCallSection()
-                .addHeaders(Map.of(
-                    CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader,
-                    CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
-                ));
+            if(doSection.getApiCallSection().getApi().startsWith("cat")){
+//                doSection.getApiCallSection()
+//                    .addHeaders(Map.of(
+//                        CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
+//                    ));
+            }else{
+                doSection.getApiCallSection()
+                    .addHeaders(Map.of(
+                        CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader,
+                        CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
+                    ));
+            }
+
         };
 }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 7b62f57d43d3f..7044a5bf5e639 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -164,7 +164,8 @@ private void validateAcceptContentTypeHeaders() {
                 return;
             }
 
-        } else if ( previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion) || acceptVersion == null) {
+        } else if ( previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion)
+            || acceptVersion == null || acceptVersion.equals("*/*")) {
             return;
         }
         throw new CompatibleApiHeadersCombinationException(

From 1d57fe78643b6476bc3352e69301c5d494fc098b Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 9 Apr 2020 08:21:13 +0200
Subject: [PATCH 38/51] precommit

---
 .../rest/compat/AbstractCompatRestTest.java   | 28 +++++++++++--------
 .../yaml/ClientYamlTestExecutionContext.java  |  3 +-
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index c67901f3720fe..afefdf7cf8254 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -78,25 +78,29 @@ private static void mutateTestCandidate(ClientYamlTestCandidate testCandidate) {
     private static Consumer<? super ExecutableSection> updateDoSection() {
         return ds -> {
             DoSection doSection = (DoSection) ds;
-            //TODO: be more selective here
+            // TODO: be more selective here
             doSection.setIgnoreWarnings(true);
 
             String compatibleHeader = createCompatibleHeader();
-            if(doSection.getApiCallSection().getApi().startsWith("cat")){
-//                doSection.getApiCallSection()
-//                    .addHeaders(Map.of(
-//                        CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
-//                    ));
-            }else{
+            if (doSection.getApiCallSection().getApi().startsWith("cat")) {
+                // doSection.getApiCallSection()
+                // .addHeaders(Map.of(
+                // CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
+                // ));
+            } else {
                 doSection.getApiCallSection()
-                    .addHeaders(Map.of(
-                        CompatibleConstants.COMPATIBLE_ACCEPT_HEADER, compatibleHeader,
-                        CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
-                    ));
+                    .addHeaders(
+                        Map.of(
+                            CompatibleConstants.COMPATIBLE_ACCEPT_HEADER,
+                            compatibleHeader,
+                            CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER,
+                            compatibleHeader
+                        )
+                    );
             }
 
         };
-}
+    }
 
     private static String createCompatibleHeader() {
         return "application/vnd.elasticsearch+json;compatible-with=" + CompatibleConstants.COMPATIBLE_VERSION;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
index c48bcdd653a39..0f310781d2367 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
@@ -144,7 +144,8 @@ private HttpEntity createEntity(List<Map<String, Object>> bodies, Map<String, St
                 bytes[position++] = xContentType.xContent().streamSeparator();
             }
             //TODO clean this up
-            ByteArrayEntity byteArrayEntity = new ByteArrayEntity(bytes, ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
+            ByteArrayEntity byteArrayEntity =
+                new ByteArrayEntity(bytes, ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
             if(byteArrayEntity.getContentType() != null){
                 byteArrayEntity.setContentType(headers.get(HTTP.CONTENT_TYPE));
             }

From 0662ef35f064417eccaa23e92ecf9b2eea27a48c Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 9 Apr 2020 12:43:38 +0200
Subject: [PATCH 39/51] cleanup

---
 .../org/elasticsearch/rest/RestRequest.java   | 103 ++++++++++--------
 .../CompatibleHeaderCombinationTests.java     |  23 ++--
 2 files changed, 76 insertions(+), 50 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 7044a5bf5e639..7cf2d9e787328 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -139,7 +139,6 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
     }
 
     private void addCompatibleParameter() {
-        validateAcceptContentTypeHeaders();
         if (isRequestCompatible()) {
             String compatibleVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
             params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, compatibleVersion);
@@ -148,59 +147,77 @@ private void addCompatibleParameter() {
         }
     }
 
-    private void validateAcceptContentTypeHeaders() {
+    private boolean isRequestCompatible() {
         String currentVersion = String.valueOf(Version.CURRENT.major);
         String previousVersion = String.valueOf(Version.CURRENT.major - 1);
-        String acceptVersion = getVersion(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER);
-        String contentTypeVersion = getVersion(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
 
-        if (hasContent()) {
-            if ((previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion) ) &&
-                acceptVersion.equals(contentTypeVersion)) {
-                return;
-            }
-            if(XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER)) == null &&
-                XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER)) == null){
-                return;
-            }
+        String acceptHeader = header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER);
+        String acceptVersion = XContentType.parseVersion(acceptHeader);
+        String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
+        String contentTypeVersion = XContentType.parseVersion(contentTypeHeader);
 
-        } else if ( previousVersion.equals(acceptVersion) || currentVersion.equals(acceptVersion)
-            || acceptVersion == null || acceptVersion.equals("*/*")) {
-            return;
-        }
-        throw new CompatibleApiHeadersCombinationException(
-            String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                    "Accept=%s Content-Type=%s hasContent=%b %s %s %s", header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER),
-                header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER), hasContent(),path(), params.toString(),method().toString()));
-    }
+        boolean isSupportedMediaTypeAccept = XContentType.fromMediaTypeOrFormat(acceptHeader) != null ||
+            acceptVersion == null;
 
-    private String getVersion(String headerName) {
-        String headerValue = header(headerName);
-        String version = XContentType.parseVersion(headerValue);
-        if(version != null){
-            return version;
-        }
-        // if version was not provided but mediaType is parsable - return Current
-        if(XContentType.fromMediaTypeOrFormat(headerValue) != null){
-            return String.valueOf(Version.CURRENT.major);
-        }
-        return headerValue;
-    }
+        boolean isSupportedMediaTypeContentType = XContentType.fromMediaTypeOrFormat(contentTypeHeader) != null ||
+            contentTypeVersion == null;
 
-    private boolean isRequestCompatible() {
         if (hasContent()) {
-            return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER)) &&
-                isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER));
+            //both headers versioned
+            if (acceptVersion != null && contentTypeVersion != null) {
+                // both Accept and Content-Type are versioned and set to a previous version
+                if (previousVersion.equals(acceptVersion) && previousVersion.equals(contentTypeVersion)) {
+                    return true;
+                }
+                // both Accept and Content-Type are versioned to a current version
+                if (currentVersion.equals(acceptVersion) && currentVersion.equals(contentTypeVersion)) {
+                    return false;
+                }
+                // both headers are versioned but set to incorrect version
+                throw new CompatibleApiHeadersCombinationException(
+                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
+                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                        contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
+            }
+
+            // Content type is set but accept is not present. It will be defaulted to JSON
+            if(isSupportedMediaTypeContentType &&
+                (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
+                return false;
+            }
+            // both Accept and Content-Type are not a compatible format, but are supported and the same and not empty
+            if (isSupportedMediaTypeContentType && isSupportedMediaTypeAccept &&
+                acceptHeader != null && contentTypeHeader != null &&
+                acceptHeader.equals(contentTypeHeader)) {
+                return false;
+            }
+        } else {
+            if (acceptVersion != null) {
+                //Accept header is versioned and set to previous
+                if (previousVersion.equals(acceptVersion)) {
+                    return true;
+                }
+                if (currentVersion.equals(acceptVersion)) {
+                    return false;
+                }
+                // Accept header is versioned but set to incorrect version
+                throw new CompatibleApiHeadersCombinationException(
+                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
+                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                        contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
+            }
+            //Accept header is not versioned but is supported
+            if (isSupportedMediaTypeAccept) {
+                return false;
+            }
         }
-        return isHeaderCompatible(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
-    }
 
-    private boolean isHeaderCompatible(String headerValue) {
-        String version = XContentType.parseVersion(headerValue);
-        return CompatibleConstants.COMPATIBLE_VERSION.equals(version);
+        throw new CompatibleApiHeadersCombinationException(
+            String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
+                    "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
     }
 
-
     private static Map<String, String> params(final String uri) {
         final Map<String, String> params = new HashMap<>();
         int index = uri.indexOf('?');
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 238326dc57f02..2cec6c5f71f88 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -39,9 +39,14 @@
 public class CompatibleHeaderCombinationTests extends ESTestCase {
     int CURRENT_VERSION = Version.CURRENT.major;
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
+    int OBSOLETE_VERSION = Version.CURRENT.major - 2;
 
-    public void testOld(){
-        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
+
+    public void testObsoleteVersion(){
+        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(null), bodyNotPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
     }
     public void testAcceptAndContentTypeCombinations() {
@@ -73,14 +78,15 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
+        // accept header when null is being defaulted to JSON. but it won't be compatible
         createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+            expect(requestCreated(), not(isCompatible())));
+
+        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
+            expect(requestCreated(), not(isCompatible())));
 
         //tests when body NOT present and one of the headers missing
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent(),
@@ -105,7 +111,10 @@ public void testAcceptAndContentTypeCombinations() {
     }
 
     public void testMediaTypeCombinations(){
-
+        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
 //        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/smile"), bodyPresent(),
 //            expect(requestCreated(), not(isCompatible())));
 

From 86f8bc0e8d3d6f2690edb472e7679dcd073861a1 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 9 Apr 2020 13:15:00 +0200
Subject: [PATCH 40/51] cleanup

---
 .../elasticsearch/rest/compat/AbstractCompatRestTest.java | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
index afefdf7cf8254..36bfd8b56c45f 100644
--- a/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
+++ b/qa/rest-compat-tests/src/main/java/org/elasticsearch/rest/compat/AbstractCompatRestTest.java
@@ -82,12 +82,8 @@ private static Consumer<? super ExecutableSection> updateDoSection() {
             doSection.setIgnoreWarnings(true);
 
             String compatibleHeader = createCompatibleHeader();
-            if (doSection.getApiCallSection().getApi().startsWith("cat")) {
-                // doSection.getApiCallSection()
-                // .addHeaders(Map.of(
-                // CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER, compatibleHeader
-                // ));
-            } else {
+            // for cat apis accept headers would break tests which expect txt response
+            if (doSection.getApiCallSection().getApi().startsWith("cat") == false) {
                 doSection.getApiCallSection()
                     .addHeaders(
                         Map.of(

From a4f472dbc22f21b1d7a6f102f273bd1bd08a6f07 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 9 Apr 2020 14:58:49 +0200
Subject: [PATCH 41/51] fix docs

---
 .../rest/compat/version7/RestGetActionV7.java    |  5 -----
 .../rest/compat/version7/RestIndexActionV7.java  |  9 ---------
 .../yaml/ClientYamlTestExecutionContext.java     | 16 ++++++++--------
 3 files changed, 8 insertions(+), 22 deletions(-)

diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
index db832ea8c7a5c..d8d01f42698f7 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestGetActionV7.java
@@ -37,11 +37,6 @@ public class RestGetActionV7 extends RestGetAction {
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class));
     static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
         + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
-    @Override
-    public String getName() {
-        return "document_get_action_v7";
-    }
-
 
     @Override
     public String getName() {
diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
index 0f407b29d11d1..51d4a56baa8fa 100644
--- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
+++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/compat/version7/RestIndexActionV7.java
@@ -80,11 +80,6 @@ public String getName() {
             return "document_create_action_v7";
         }
 
-        @Override
-        public String getName() {
-            return "document_create_action_v7";
-        }
-
         @Override
         public List<Route> routes() {
             return unmodifiableList(
@@ -106,10 +101,6 @@ public boolean compatibilityRequired() {
     }
 
     public static final class CompatibleAutoIdHandler extends RestIndexAction.AutoIdHandler {
-        @Override
-        public String getName() {
-            return "document_create_action_auto_id_v7";
-        }
 
         public CompatibleAutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
             super(nodesInCluster);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
index 0f310781d2367..de36c25f2998b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
@@ -119,12 +119,9 @@ private HttpEntity createEntity(List<Map<String, Object>> bodies, Map<String, St
         if (bodies.size() == 1) {
             XContentType xContentType = getContentType(headers, XContentType.values());
             BytesRef bytesRef = bodyAsBytesRef(bodies.get(0), xContentType);
-            //TODO clean this up
             ByteArrayEntity byteArrayEntity = new ByteArrayEntity(bytesRef.bytes, bytesRef.offset, bytesRef.length,
                 ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
-            if(byteArrayEntity.getContentType() != null){
-                byteArrayEntity.setContentType(headers.get(HTTP.CONTENT_TYPE));
-            }
+            overrideContentType(headers, byteArrayEntity);
             return byteArrayEntity;
         } else {
             XContentType xContentType = getContentType(headers, STREAMING_CONTENT_TYPES);
@@ -143,16 +140,19 @@ private HttpEntity createEntity(List<Map<String, Object>> bodies, Map<String, St
                 }
                 bytes[position++] = xContentType.xContent().streamSeparator();
             }
-            //TODO clean this up
             ByteArrayEntity byteArrayEntity =
                 new ByteArrayEntity(bytes, ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
-            if(byteArrayEntity.getContentType() != null){
-                byteArrayEntity.setContentType(headers.get(HTTP.CONTENT_TYPE));
-            }
+            overrideContentType(headers, byteArrayEntity);
             return byteArrayEntity;
         }
     }
 
+    private void overrideContentType(Map<String, String> headers, ByteArrayEntity byteArrayEntity) {
+        if (byteArrayEntity.getContentType() != null && headers.get(HTTP.CONTENT_TYPE) != null) {
+            byteArrayEntity.setContentType(headers.get(HTTP.CONTENT_TYPE));
+        }
+    }
+
     private XContentType getContentType(Map<String, String> headers, XContentType[] supportedContentTypes) {
         XContentType xContentType = null;
         String contentType = headers.get("Content-Type");

From bb827bcad0e5a93e434763fe59d4822da941471e Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 9 Apr 2020 17:47:48 +0200
Subject: [PATCH 42/51] fix different mimetypes

---
 .../http/AbstractHttpServerTransport.java     |  4 +-
 .../org/elasticsearch/rest/RestRequest.java   |  9 ++-
 .../CompatibleHeaderCombinationTests.java     | 43 +++++++++-----
 .../rest/RestControllerTests.java             | 31 ----------
 .../elasticsearch/rest/RestRequestTests.java  | 57 -------------------
 5 files changed, 35 insertions(+), 109 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
index dee9325f8827e..431201512ddda 100644
--- a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
+++ b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
@@ -354,7 +354,9 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan
                 innerRestRequest = RestRequest.requestWithoutParameters(xContentRegistry, httpRequest, httpChannel);
             } catch (final RestRequest.CompatibleApiHeadersCombinationException e){
                 badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e);
-                innerRestRequest = requestWithoutContentTypeHeader(httpRequest, httpChannel, badRequestCause);
+                //todo // tempting to just rethrow. removing content type is just making this less obvious
+                throw e;
+//                innerRestRequest = requestWithoutContentTypeHeader(httpRequest, httpChannel, badRequestCause);
             }
             restRequest = innerRestRequest;
         }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 7cf2d9e787328..490181f32c333 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -157,10 +157,10 @@ private boolean isRequestCompatible() {
         String contentTypeVersion = XContentType.parseVersion(contentTypeHeader);
 
         boolean isSupportedMediaTypeAccept = XContentType.fromMediaTypeOrFormat(acceptHeader) != null ||
-            acceptVersion == null;
+            acceptHeader == null;
 
         boolean isSupportedMediaTypeContentType = XContentType.fromMediaTypeOrFormat(contentTypeHeader) != null ||
-            contentTypeVersion == null;
+            contentTypeHeader == null;
 
         if (hasContent()) {
             //both headers versioned
@@ -185,10 +185,9 @@ private boolean isRequestCompatible() {
                 (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
                 return false;
             }
-            // both Accept and Content-Type are not a compatible format, but are supported and the same and not empty
+            // both Accept and Content-Type are not a compatible format, but are supported and not empty
             if (isSupportedMediaTypeContentType && isSupportedMediaTypeAccept &&
-                acceptHeader != null && contentTypeHeader != null &&
-                acceptHeader.equals(contentTypeHeader)) {
+                acceptHeader != null && contentTypeHeader != null) {
                 return false;
             }
         } else {
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 2cec6c5f71f88..8883421e0c1b7 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -41,14 +41,6 @@ public class CompatibleHeaderCombinationTests extends ESTestCase {
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
     int OBSOLETE_VERSION = Version.CURRENT.major - 2;
 
-
-    public void testObsoleteVersion(){
-        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-
-        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(null), bodyNotPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-    }
     public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(requestCreated(), isCompatible()));
@@ -110,18 +102,39 @@ public void testAcceptAndContentTypeCombinations() {
             expect(requestCreated(), not(isCompatible())));
     }
 
-    public void testMediaTypeCombinations(){
+    public void testObsoleteVersion() {
+        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+
+        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(null), bodyNotPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+    }
+
+    public void testMediaTypeCombinations() {
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
-//        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/smile"), bodyPresent(),
-//            expect(requestCreated(), not(isCompatible())));
 
-//        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
-//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-//        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
-//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        //this is for instance used by SQL
+        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/cbor"), bodyPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+        createRequestWith(acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"),
+            contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=7"), bodyPresent(),
+            expect(requestCreated(), isCompatible()));
+
+        //different versions on different media types
+        createRequestWith(acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"),
+            contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=8"), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+    }
+
+    public void testMalfunctionedMediaTypes(){
+        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
     }
 
     private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends Exception> exceptionClass) {
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index dafbacef28f35..4811dbc466623 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.usage.UsageService;
-import org.junit.Assert;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -645,36 +644,6 @@ private String randomCompatibleMimeType(byte version) {
         return randomFrom("application/vnd.elasticsearch+" + subtype + ";compatible-with=" + version);
     }
 
-    public void testIncorrectCompatibleHandlersDoNotDispatch() {
-        final byte version = (byte) (Version.CURRENT.major - 1);
-        final String mimeType = randomCompatibleMimeType(version);
-        String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
-        final List<String> mimeTypeList = Collections.singletonList(mimeType);
-
-        // request to compatible api with body requires a content-type header.
-        // this one does no have it, making it non compatible
-        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
-            .withContent(new BytesArray(content), RestRequest.parseContentType(mimeTypeList))
-            .withPath("/foo")
-            .withHeaders(Map.of("Accept", mimeTypeList))
-            .build();
-        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
-            @Override
-            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-                Assert.fail();
-            }
-
-            @Override
-            public boolean compatibilityRequired() {
-                return true;
-            }
-        });
-        AssertingChannel channel = new AssertingChannel(fakeRestRequest, false, RestStatus.BAD_REQUEST);
-        assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
-        assertTrue(channel.getSendResponseCalled());
-    }
-
     private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
         HttpServerTransport {
 
diff --git a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
index 5a5b2c924c65f..487bbed5a5999 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java
@@ -43,7 +43,6 @@
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -246,62 +245,6 @@ public void testRequiredContent() {
         assertEquals("unknown content type", e.getMessage());
     }
 
-    public void testCorrectCompatibleRequest() {
-        List<String> compatibleHeaderValue = List.of("application/vnd.elasticsearch+json;compatible-with=7");
-
-        // when no body - only Accept header is required
-        FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        builder.withHeaders(Map.of("Accept", compatibleHeaderValue));
-        RestRequest restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
-
-        // when no body - both Accept and Content-Type provided, but only Accept required - still OK
-        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        builder.withHeaders(Map.of("Accept", compatibleHeaderValue, "Content-Type", compatibleHeaderValue));
-        builder.withContent(new BytesArray("some content"), null);
-        restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
-
-
-        // with body - both Accept and Content-Type are required
-        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        builder.withHeaders(Map.of("Accept", compatibleHeaderValue, "Content-Type", compatibleHeaderValue));
-        builder.withContent(new BytesArray("some content"), null);
-        restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), equalTo("7"));
-    }
-
-    public void testIncorrectCompatibleRequest() {
-        List<String> compatibleHeaderValue = List.of("application/vnd.elasticsearch+json;compatible-with=7");
-
-        // no body, but no Accept header - won't set compatible param
-        FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        RestRequest restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
-
-        // no body, but incorrect header value
-        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        builder.withHeaders(Map.of("Accept", List.of("application/json")));
-        restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
-
-        // body provided, but no Content-Type
-        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        builder.withHeaders(Map.of("Accept", compatibleHeaderValue));
-        builder.withContent(new BytesArray("some content"), null);
-        restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
-
-
-        // body provided, but incorrect Content-Type
-        builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
-        builder.withHeaders(Map.of("Accept", compatibleHeaderValue, "Content-Type", List.of("application/json")));
-        builder.withContent(new BytesArray("some content"), null);
-        restRequest = builder.build();
-        assertThat(restRequest.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY), nullValue());
-
-    }
-
     private static RestRequest contentRestRequest(String content, Map<String, String> params) {
         Map<String, List<String>> headers = new HashMap<>();
         headers.put("Content-Type", Collections.singletonList("application/json"));

From e9a4fbd9dc0b75af2d6c2621f0d3357074aaf81e Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 10 Apr 2020 09:18:00 +0200
Subject: [PATCH 43/51] support text formats

---
 .../common/xcontent/XContentType.java         | 14 ++++++-------
 .../org/elasticsearch/rest/RestRequest.java   | 12 +++++++----
 .../common/xcontent/XContentTypeTests.java    | 17 +++++++++++++++
 .../CompatibleHeaderCombinationTests.java     | 21 +++++++++++++++++++
 .../xpack/sql/plugin/TextFormat.java          |  1 +
 5 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index 5378c3c46eb79..8846cbeb1daa2 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -117,7 +117,7 @@ public XContent xContent() {
     };
 
     private static final Pattern COMPATIBLE_API_HEADER_PATTERN = Pattern.compile(
-        "application/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?",
+        "(application|text)/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?",
         Pattern.CASE_INSENSITIVE);
 
     /**
@@ -138,7 +138,7 @@ public static XContentType fromMediaTypeOrFormat(String mediaTypeHeaderValue) {
             }
         }
         final String lowercaseMediaType = mediaType.toLowerCase(Locale.ROOT);
-        if (lowercaseMediaType.startsWith("application/*")) {
+        if (lowercaseMediaType.startsWith("application/*") || lowercaseMediaType.equals("*/*")) {
             return JSON;
         }
 
@@ -167,12 +167,12 @@ public static XContentType fromMediaType(String mediaTypeHeaderValue) {
         return null;
     }
 
-    //default scope for testing
-    static String parseMediaType(String mediaType) {
+    //public scope needed for text formats hack
+    public static String parseMediaType(String mediaType) {
         if (mediaType != null) {
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
             if (matcher.find()) {
-                return "application/" + matcher.group(2).toLowerCase(Locale.ROOT);
+                return matcher.group(1) + "/" + matcher.group(3).toLowerCase(Locale.ROOT);
             }
         }
 
@@ -182,9 +182,9 @@ static String parseMediaType(String mediaType) {
     public static String parseVersion(String mediaType){
         if(mediaType != null){
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
-            if (matcher.find() && "vnd.elasticsearch+".equalsIgnoreCase(matcher.group(1))) {
+            if (matcher.find() && "vnd.elasticsearch+".equalsIgnoreCase(matcher.group(2))) {
 
-                return matcher.group(4);
+                return matcher.group(5);
             }
         }
         return null;
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 490181f32c333..aab5d3fe86a25 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -146,6 +146,7 @@ private void addCompatibleParameter() {
             param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
         }
     }
+    Set<String> textFormats = Set.of("text/plain", "text/csv", "text/tab-separated-values");
 
     private boolean isRequestCompatible() {
         String currentVersion = String.valueOf(Version.CURRENT.major);
@@ -156,11 +157,14 @@ private boolean isRequestCompatible() {
         String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
         String contentTypeVersion = XContentType.parseVersion(contentTypeHeader);
 
-        boolean isSupportedMediaTypeAccept = XContentType.fromMediaTypeOrFormat(acceptHeader) != null ||
-            acceptHeader == null;
+        boolean isSupportedMediaTypeAccept = acceptHeader == null ||
+            XContentType.fromMediaTypeOrFormat(acceptHeader) != null ||
+            // should we care what media type was used? we don't know about text formats here. hence the Set<String> textFormats
+            textFormats.contains(XContentType.parseMediaType(acceptHeader).toLowerCase(Locale.ROOT));
 
-        boolean isSupportedMediaTypeContentType = XContentType.fromMediaTypeOrFormat(contentTypeHeader) != null ||
-            contentTypeHeader == null;
+        boolean isSupportedMediaTypeContentType = contentTypeHeader == null ||
+            XContentType.fromMediaTypeOrFormat(contentTypeHeader) != null ||
+           textFormats.contains(XContentType.parseMediaType(contentTypeHeader).toLowerCase(Locale.ROOT));
 
         if (hasContent()) {
             //both headers versioned
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
index 856e544a23055..3ceb93ef6abec 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java
@@ -121,4 +121,21 @@ public void testVersionParsing() {
         assertThat(XContentType.parseVersion("APPLICATION/JSON"),
             nullValue());
     }
+
+    public void testVersionParsingOnText() {
+        String version = String.valueOf(Math.abs(randomByte()));
+        assertThat(XContentType.parseVersion("text/vnd.elasticsearch+csv;compatible-with=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("text/vnd.elasticsearch+text;compatible-with=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("text/vnd.elasticsearch+tab-separated-values;compatible-with=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("text/csv"),
+            nullValue());
+
+        assertThat(XContentType.parseVersion("TEXT/VND.ELASTICSEARCH+CSV;COMPATIBLE-WITH=" + version),
+            equalTo(version));
+        assertThat(XContentType.parseVersion("TEXT/csv"),
+            nullValue());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 8883421e0c1b7..085a88027b306 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -115,6 +115,8 @@ public void testMediaTypeCombinations() {
             expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader("*/*"), contentTypeHeader("application/json"), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
 
         //this is for instance used by SQL
         createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/cbor"), bodyPresent(),
@@ -130,6 +132,25 @@ public void testMediaTypeCombinations() {
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
     }
 
+    public void testTextMediaTypes(){
+        createRequestWith(acceptHeader("text/tab-separated-values"), contentTypeHeader("application/json"), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader("text/plain"), contentTypeHeader("application/json"), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader("text/csv"), contentTypeHeader("application/json"), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+        //versioned
+        createRequestWith(acceptHeader("text/vnd.elasticsearch+tab-separated-values;compatible-with=7"),
+            contentTypeHeader(7), bodyNotPresent(),
+            expect(requestCreated(), isCompatible()));
+        createRequestWith(acceptHeader("text/vnd.elasticsearch+plain;compatible-with=7"),
+            contentTypeHeader(7), bodyNotPresent(),
+            expect(requestCreated(), isCompatible()));
+        createRequestWith(acceptHeader("text/vnd.elasticsearch+csv;compatible-with=7"),
+            contentTypeHeader(7), bodyNotPresent(),
+            expect(requestCreated(), isCompatible()));
+    }
     public void testMalfunctionedMediaTypes(){
         createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java
index dd837e65315aa..bde664f4ad4af 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java
@@ -258,6 +258,7 @@ boolean hasHeader(RestRequest request) {
     }
 
     static TextFormat fromMediaTypeOrFormat(String accept) {
+        //TODO we should include version parsing here too. This and XContentType should be unified somehow..
         for (TextFormat text : values()) {
             String contentType = text.contentType();
             if (contentType.equalsIgnoreCase(accept)

From f876d468105c7b1923c35a488e0977dbe85f2fd5 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 10 Apr 2020 09:42:18 +0200
Subject: [PATCH 44/51] do not validate media type correctness

---
 .../java/org/elasticsearch/rest/RestRequest.java  | 15 ++++++++-------
 .../rest/CompatibleHeaderCombinationTests.java    | 12 ++++++------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index aab5d3fe86a25..43e0e65904176 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -146,7 +146,7 @@ private void addCompatibleParameter() {
             param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
         }
     }
-    Set<String> textFormats = Set.of("text/plain", "text/csv", "text/tab-separated-values");
+//    Set<String> textFormats = Set.of("text/plain", "text/csv", "text/tab-separated-values");
 
     private boolean isRequestCompatible() {
         String currentVersion = String.valueOf(Version.CURRENT.major);
@@ -157,14 +157,15 @@ private boolean isRequestCompatible() {
         String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
         String contentTypeVersion = XContentType.parseVersion(contentTypeHeader);
 
-        boolean isSupportedMediaTypeAccept = acceptHeader == null ||
-            XContentType.fromMediaTypeOrFormat(acceptHeader) != null ||
+        boolean isSupportedMediaTypeAccept = acceptHeader == null || XContentType.parseMediaType(acceptHeader) != null;
             // should we care what media type was used? we don't know about text formats here. hence the Set<String> textFormats
-            textFormats.contains(XContentType.parseMediaType(acceptHeader).toLowerCase(Locale.ROOT));
 
-        boolean isSupportedMediaTypeContentType = contentTypeHeader == null ||
-            XContentType.fromMediaTypeOrFormat(contentTypeHeader) != null ||
-           textFormats.contains(XContentType.parseMediaType(contentTypeHeader).toLowerCase(Locale.ROOT));
+//            XContentType.fromMediaTypeOrFormat(acceptHeader) != null ||
+//            textFormats.contains(XContentType.parseMediaType(acceptHeader).toLowerCase(Locale.ROOT));
+
+        boolean isSupportedMediaTypeContentType = contentTypeHeader == null || XContentType.parseMediaType(contentTypeHeader) != null;
+//            XContentType.fromMediaTypeOrFormat(contentTypeHeader) != null ||
+//           textFormats.contains(XContentType.parseMediaType(contentTypeHeader).toLowerCase(Locale.ROOT));
 
         if (hasContent()) {
             //both headers versioned
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 085a88027b306..43cce035fd650 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -151,12 +151,12 @@ public void testTextMediaTypes(){
             contentTypeHeader(7), bodyNotPresent(),
             expect(requestCreated(), isCompatible()));
     }
-    public void testMalfunctionedMediaTypes(){
-        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-    }
+//    public void testMalfunctionedMediaTypes(){
+//        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
+//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+//        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
+//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+//    }
 
     private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends Exception> exceptionClass) {
         return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {

From 47fd5cb905e6a6a31968de6025267b22dd7c9cbf Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 10 Apr 2020 11:36:51 +0200
Subject: [PATCH 45/51] do not allow empty accept header

---
 .../src/main/java/org/elasticsearch/rest/RestRequest.java | 8 ++++----
 .../rest/CompatibleHeaderCombinationTests.java            | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 43e0e65904176..3ccf3633f7207 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -186,10 +186,10 @@ private boolean isRequestCompatible() {
             }
 
             // Content type is set but accept is not present. It will be defaulted to JSON
-            if(isSupportedMediaTypeContentType &&
-                (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
-                return false;
-            }
+//            if(isSupportedMediaTypeContentType &&
+//                (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
+//                return false;
+//            }
             // both Accept and Content-Type are not a compatible format, but are supported and not empty
             if (isSupportedMediaTypeContentType && isSupportedMediaTypeAccept &&
                 acceptHeader != null && contentTypeHeader != null) {
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 43cce035fd650..9c44afa0d0c73 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -73,12 +73,12 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-        // accept header when null is being defaulted to JSON. but it won't be compatible
+        // with body present - both headers have to be versioned
         createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
         //tests when body NOT present and one of the headers missing
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent(),

From bcb11fddf07df33c24fb05b4d3dc9e47aaa9c5b6 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Fri, 10 Apr 2020 13:17:56 +0200
Subject: [PATCH 46/51] allow empty accept header

---
 .../main/java/org/elasticsearch/Version.java  |  4 +-
 .../org/elasticsearch/rest/RestRequest.java   | 44 +++++++++----------
 .../CompatibleHeaderCombinationTests.java     | 14 ++++--
 3 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index 83c1b4b7e227f..fbdcbe45e97ba 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -80,6 +80,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_7_8_0 = new Version(7080099, org.apache.lucene.util.Version.LUCENE_8_5_0);
     public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_5_0);
     public static final Version CURRENT = V_8_0_0;
+    public static final Version PREVIOUS = V_7_0_0;
 
     private static final ImmutableOpenIntMap<Version> idToVersion;
 
@@ -89,7 +90,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
         for (final Field declaredField : Version.class.getFields()) {
             if (declaredField.getType().equals(Version.class)) {
                 final String fieldName = declaredField.getName();
-                if (fieldName.equals("CURRENT") || fieldName.equals("V_EMPTY")) {
+                if (fieldName.equals("CURRENT") || fieldName.equals("V_EMPTY") || fieldName.equals("PREVIOUS")) {
                     continue;
                 }
                 assert fieldName.matches("V_\\d+_\\d+_\\d+")
@@ -402,6 +403,7 @@ public static List<Version> getDeclaredVersions(final Class<?> versionClass) {
             }
             switch (field.getName()) {
                 case "CURRENT":
+                case "PREVIOUS":
                 case "V_EMPTY":
                     continue;
             }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 3ccf3633f7207..9efc6489ae13e 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -139,16 +139,15 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
     }
 
     private void addCompatibleParameter() {
-        if (isRequestCompatible()) {
-            String compatibleVersion = XContentType.parseVersion(header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER));
-            params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, compatibleVersion);
+        Version compatible = compatibleWithVersion();
+        if (Version.PREVIOUS.equals(compatible)) {
+            params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, String.valueOf(compatible.major));
             //use it so it won't fail request validation with unused parameter
             param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
         }
     }
-//    Set<String> textFormats = Set.of("text/plain", "text/csv", "text/tab-separated-values");
 
-    private boolean isRequestCompatible() {
+    private Version compatibleWithVersion() {
         String currentVersion = String.valueOf(Version.CURRENT.major);
         String previousVersion = String.valueOf(Version.CURRENT.major - 1);
 
@@ -157,26 +156,21 @@ private boolean isRequestCompatible() {
         String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
         String contentTypeVersion = XContentType.parseVersion(contentTypeHeader);
 
+        //TODO not sure about this one as this does not cover text formats
         boolean isSupportedMediaTypeAccept = acceptHeader == null || XContentType.parseMediaType(acceptHeader) != null;
-            // should we care what media type was used? we don't know about text formats here. hence the Set<String> textFormats
-
-//            XContentType.fromMediaTypeOrFormat(acceptHeader) != null ||
-//            textFormats.contains(XContentType.parseMediaType(acceptHeader).toLowerCase(Locale.ROOT));
 
         boolean isSupportedMediaTypeContentType = contentTypeHeader == null || XContentType.parseMediaType(contentTypeHeader) != null;
-//            XContentType.fromMediaTypeOrFormat(contentTypeHeader) != null ||
-//           textFormats.contains(XContentType.parseMediaType(contentTypeHeader).toLowerCase(Locale.ROOT));
 
         if (hasContent()) {
             //both headers versioned
             if (acceptVersion != null && contentTypeVersion != null) {
                 // both Accept and Content-Type are versioned and set to a previous version
                 if (previousVersion.equals(acceptVersion) && previousVersion.equals(contentTypeVersion)) {
-                    return true;
+                    return Version.PREVIOUS;
                 }
                 // both Accept and Content-Type are versioned to a current version
                 if (currentVersion.equals(acceptVersion) && currentVersion.equals(contentTypeVersion)) {
-                    return false;
+                    return Version.CURRENT;
                 }
                 // both headers are versioned but set to incorrect version
                 throw new CompatibleApiHeadersCombinationException(
@@ -184,25 +178,29 @@ private boolean isRequestCompatible() {
                             "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
                         contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
             }
-
-            // Content type is set but accept is not present. It will be defaulted to JSON
-//            if(isSupportedMediaTypeContentType &&
-//                (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
-//                return false;
-//            }
+            // Content-Type is versioned but accept is not present
+            if(previousVersion.equals(contentTypeVersion)
+                && (acceptHeader == null || acceptHeader.equals("*/*") )) {
+                return Version.PREVIOUS;
+            }
+            // Content type is set (not verioned) but accept is not present. It will be defaulted to JSON
+            if(isSupportedMediaTypeContentType &&
+                (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
+                return Version.CURRENT;
+            }
             // both Accept and Content-Type are not a compatible format, but are supported and not empty
             if (isSupportedMediaTypeContentType && isSupportedMediaTypeAccept &&
                 acceptHeader != null && contentTypeHeader != null) {
-                return false;
+                return Version.CURRENT;
             }
         } else {
             if (acceptVersion != null) {
                 //Accept header is versioned and set to previous
                 if (previousVersion.equals(acceptVersion)) {
-                    return true;
+                    return Version.PREVIOUS;
                 }
                 if (currentVersion.equals(acceptVersion)) {
-                    return false;
+                    return Version.CURRENT;
                 }
                 // Accept header is versioned but set to incorrect version
                 throw new CompatibleApiHeadersCombinationException(
@@ -212,7 +210,7 @@ private boolean isRequestCompatible() {
             }
             //Accept header is not versioned but is supported
             if (isSupportedMediaTypeAccept) {
-                return false;
+                return Version.CURRENT;
             }
         }
 
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 9c44afa0d0c73..0a14c3628146d 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -41,6 +41,14 @@ public class CompatibleHeaderCombinationTests extends ESTestCase {
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
     int OBSOLETE_VERSION = Version.CURRENT.major - 2;
 
+//    public void testX(){
+//
+//        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
+//            expect(requestCreated(), isCompatible()));
+////        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
+////            expect(requestCreated(), isCompatible()));
+//
+//    }
     public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(requestCreated(), isCompatible()));
@@ -73,12 +81,12 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-        // with body present - both headers have to be versioned
+        // with body present and contentType versioned - default the accept header version to the one from content-type
         createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+            expect(requestCreated(), not(isCompatible())));
 
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+            expect(requestCreated(), isCompatible()));
 
         //tests when body NOT present and one of the headers missing
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent(),

From 225a00ba2e4fc9f824da555444d2fbab04c046e0 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 14 Apr 2020 16:32:50 +0200
Subject: [PATCH 47/51] exception propagation to user

---
 .../rest/Netty4BadRequestIT.java              | 21 ++++++++
 .../main/java/org/elasticsearch/Version.java  |  5 +-
 .../http/AbstractHttpServerTransport.java     |  9 ++--
 .../org/elasticsearch/rest/RestRequest.java   | 53 ++++++++++++++-----
 4 files changed, 69 insertions(+), 19 deletions(-)

diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
index cfda71f10096e..cc40163ea6ad0 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
@@ -101,4 +101,25 @@ public void testInvalidHeaderValue() throws IOException {
         assertThat(map.get("type"), equalTo("content_type_header_exception"));
         assertThat(map.get("reason"), equalTo("java.lang.IllegalArgumentException: invalid Content-Type header []"));
     }
+
+    public void testInvalidHeaderCombinations() throws IOException {
+        final Request request = new Request("GET", "/_cluster/settings");
+        final RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Content-Type", "application/vnd.elasticsearch+json;compatible-with=7");
+        options.addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=8");
+        request.setOptions(options);
+        request.setJsonEntity("{\"transient\":{\"search.*\":null}}");
+
+        final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        final Response response = e.getResponse();
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(400));
+        final ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        final Map<String, Object> map = objectPath.evaluate("error");
+        assertThat(map.get("type"), equalTo("compatible_api_headers_combination_exception"));
+        assertThat(map.get("reason"), equalTo("Request with a body and incompatible Accept and Content-Type header values. " +
+            "Accept=application/vnd.elasticsearch+json;compatible-with=8 " +
+            "Content-Type=application/vnd.elasticsearch+json;compatible-with=7 " +
+            "hasContent=true " +
+            "/_cluster/settings {} GET"));
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index fbdcbe45e97ba..d1e004aecd90a 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -80,7 +80,6 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_7_8_0 = new Version(7080099, org.apache.lucene.util.Version.LUCENE_8_5_0);
     public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_5_0);
     public static final Version CURRENT = V_8_0_0;
-    public static final Version PREVIOUS = V_7_0_0;
 
     private static final ImmutableOpenIntMap<Version> idToVersion;
 
@@ -343,6 +342,10 @@ public boolean isCompatible(Version version) {
         return compatible;
     }
 
+    public static Version minimumRestCompatibilityVersion(){
+        return Version.V_7_0_0;
+    }
+
     @SuppressForbidden(reason = "System.out.*")
     public static void main(String[] args) {
         final String versionOutput = String.format(
diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
index 431201512ddda..f30acc2e3e8b7 100644
--- a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
+++ b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
@@ -354,9 +354,7 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan
                 innerRestRequest = RestRequest.requestWithoutParameters(xContentRegistry, httpRequest, httpChannel);
             } catch (final RestRequest.CompatibleApiHeadersCombinationException e){
                 badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e);
-                //todo // tempting to just rethrow. removing content type is just making this less obvious
-                throw e;
-//                innerRestRequest = requestWithoutContentTypeHeader(httpRequest, httpChannel, badRequestCause);
+                innerRestRequest = RestRequest.requestNoValidation(xContentRegistry, httpRequest, httpChannel);
             }
             restRequest = innerRestRequest;
         }
@@ -389,12 +387,11 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan
     }
 
     private RestRequest requestWithoutContentTypeHeader(HttpRequest httpRequest, HttpChannel httpChannel, Exception badRequestCause) {
-        HttpRequest httpRequestWithoutContentType = httpRequest.removeHeader("Content-Type");
         try {
-            return RestRequest.request(xContentRegistry, httpRequestWithoutContentType, httpChannel);
+            return RestRequest.requestWithoutContentType(xContentRegistry, httpRequest, httpChannel);
         } catch (final RestRequest.BadParameterException e) {
             badRequestCause.addSuppressed(e);
-            return RestRequest.requestWithoutParameters(xContentRegistry, httpRequestWithoutContentType, httpChannel);
+            return RestRequest.requestNoValidation(xContentRegistry, httpRequest, httpChannel);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index 9efc6489ae13e..a1d09e0b28127 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -88,6 +88,12 @@ protected RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String
 
     private RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path,
                         Map<String, List<String>> headers, HttpRequest httpRequest, HttpChannel httpChannel, long requestId) {
+        this(xContentRegistry, params, path, headers, httpRequest, httpChannel, requestId, true);
+
+    }
+    private RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path,
+                        Map<String, List<String>> headers, HttpRequest httpRequest, HttpChannel httpChannel,
+                        long requestId, boolean headersValidation) {
         final XContentType xContentType;
         try {
             xContentType = parseContentType(headers.get("Content-Type"));
@@ -104,7 +110,7 @@ private RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String>
         this.rawPath = path;
         this.headers = Collections.unmodifiableMap(headers);
         this.requestId = requestId;
-        addCompatibleParameter();
+        addCompatibleParameter(headersValidation);
     }
 
     protected RestRequest(RestRequest restRequest) {
@@ -138,12 +144,14 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
             requestIdGenerator.incrementAndGet());
     }
 
-    private void addCompatibleParameter() {
-        Version compatible = compatibleWithVersion();
-        if (Version.PREVIOUS.equals(compatible)) {
-            params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, String.valueOf(compatible.major));
-            //use it so it won't fail request validation with unused parameter
-            param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
+    private void addCompatibleParameter(boolean headersValidation) {
+        if(headersValidation){
+            Version compatible = compatibleWithVersion();
+            if (Version.minimumRestCompatibilityVersion().equals(compatible)) {
+                params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, String.valueOf(compatible.major));
+                //use it so it won't fail request validation with unused parameter
+                param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
+            }
         }
     }
 
@@ -166,7 +174,7 @@ private Version compatibleWithVersion() {
             if (acceptVersion != null && contentTypeVersion != null) {
                 // both Accept and Content-Type are versioned and set to a previous version
                 if (previousVersion.equals(acceptVersion) && previousVersion.equals(contentTypeVersion)) {
-                    return Version.PREVIOUS;
+                    return Version.minimumRestCompatibilityVersion();
                 }
                 // both Accept and Content-Type are versioned to a current version
                 if (currentVersion.equals(acceptVersion) && currentVersion.equals(contentTypeVersion)) {
@@ -181,9 +189,9 @@ private Version compatibleWithVersion() {
             // Content-Type is versioned but accept is not present
             if(previousVersion.equals(contentTypeVersion)
                 && (acceptHeader == null || acceptHeader.equals("*/*") )) {
-                return Version.PREVIOUS;
+                return Version.minimumRestCompatibilityVersion();
             }
-            // Content type is set (not verioned) but accept is not present. It will be defaulted to JSON
+            // Content type is set (not versioned) but accept is not present. It will be defaulted to JSON
             if(isSupportedMediaTypeContentType &&
                 (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
                 return Version.CURRENT;
@@ -197,7 +205,7 @@ private Version compatibleWithVersion() {
             if (acceptVersion != null) {
                 //Accept header is versioned and set to previous
                 if (previousVersion.equals(acceptVersion)) {
-                    return Version.PREVIOUS;
+                    return Version.minimumRestCompatibilityVersion();
                 }
                 if (currentVersion.equals(acceptVersion)) {
                     return Version.CURRENT;
@@ -255,9 +263,30 @@ public static RestRequest requestWithoutParameters(NamedXContentRegistry xConten
                                                        HttpChannel httpChannel) {
         Map<String, String> params = Collections.emptyMap();
         return new RestRequest(xContentRegistry, params, httpRequest.uri(), httpRequest.getHeaders(), httpRequest, httpChannel,
-            requestIdGenerator.incrementAndGet());
+            requestIdGenerator.incrementAndGet(), false);
+    }
+
+    public static RestRequest requestWithoutContentType(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest,
+                                                       HttpChannel httpChannel) {
+        HttpRequest httpRequestWithoutContentType = httpRequest.removeHeader("Content-Type");
+        Map<String, String> params = params(httpRequest.uri());
+        String path = path(httpRequest.uri());
+        return new RestRequest(xContentRegistry, params, path, httpRequestWithoutContentType.getHeaders(),
+            httpRequestWithoutContentType, httpChannel,
+            requestIdGenerator.incrementAndGet(), false);
+    }
+
+
+    public static RestRequest requestNoValidation(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, HttpChannel httpChannel) {
+        Map<String, String> params = Collections.emptyMap();
+        HttpRequest httpRequestWithoutContentType = httpRequest.removeHeader("Content-Type");
+
+        return new RestRequest(xContentRegistry, params, httpRequestWithoutContentType.uri(),
+            httpRequestWithoutContentType.getHeaders(), httpRequestWithoutContentType, httpChannel,
+            requestIdGenerator.incrementAndGet(), false);
     }
 
+
     public enum Method {
         GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH, TRACE, CONNECT
     }

From ebaa7c9b9c1ef60db2029b61d46e4728f7383c78 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Tue, 14 Apr 2020 20:16:01 +0200
Subject: [PATCH 48/51] check which tests fail

---
 .../org/elasticsearch/rest/RestRequest.java   | 47 +++++++++++++++++--
 .../CompatibleHeaderCombinationTests.java     | 45 +++++++++++-------
 .../test/rest/FakeRestRequest.java            | 13 +++++
 3 files changed, 83 insertions(+), 22 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index a1d09e0b28127..d32e27164e93c 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -146,14 +146,51 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
 
     private void addCompatibleParameter(boolean headersValidation) {
         if(headersValidation){
-            Version compatible = compatibleWithVersion();
-            if (Version.minimumRestCompatibilityVersion().equals(compatible)) {
-                params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY, String.valueOf(compatible.major));
+//            Version compatible = compatibleWithVersion();
+//            if (Version.minimumRestCompatibilityVersion().equals(compatible)) {
+            if(isRequestingCompatibility()) {
+                params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY,
+                    String.valueOf(Version.minimumRestCompatibilityVersion().major));
                 //use it so it won't fail request validation with unused parameter
                 param(CompatibleConstants.COMPATIBLE_PARAMS_KEY);
             }
         }
     }
+    private boolean isRequestingCompatibility() {
+        String acceptHeader = header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER);
+        String aVersion = XContentType.parseVersion(acceptHeader);
+        byte acceptVersion = aVersion == null ? Version.CURRENT.major : Integer.valueOf(aVersion).byteValue();
+        String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
+        String cVersion = XContentType.parseVersion(contentTypeHeader);
+        byte contentTypeVersion = cVersion == null ? Version.CURRENT.major : Integer.valueOf(cVersion).byteValue();
+
+        if(Version.CURRENT.major < acceptVersion || Version.CURRENT.major - acceptVersion > 1 ){
+//            throw new IllegalStateException("something about only 1 major version support");
+            throw new CompatibleApiHeadersCombinationException(
+                String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
+                        "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                    contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
+        }
+        if (hasContent()) {
+            if(Version.CURRENT.major < contentTypeVersion || Version.CURRENT.major - contentTypeVersion > 1 ){
+//                throw new IllegalStateException("something about only 1 major version support");
+                throw new CompatibleApiHeadersCombinationException(
+                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
+                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                        contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
+            }
+
+            if (contentTypeVersion != acceptVersion) {
+//                throw new IllegalStateException("something about needing to match");
+                throw new CompatibleApiHeadersCombinationException(
+                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
+                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                        contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
+            }
+        }
+
+        return contentTypeVersion < Version.CURRENT.major || acceptVersion < Version.CURRENT.major;
+    }
 
     private Version compatibleWithVersion() {
         String currentVersion = String.valueOf(Version.CURRENT.major);
@@ -277,7 +314,9 @@ public static RestRequest requestWithoutContentType(NamedXContentRegistry xConte
     }
 
 
-    public static RestRequest requestNoValidation(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, HttpChannel httpChannel) {
+    public static RestRequest requestNoValidation(NamedXContentRegistry xContentRegistry,
+                                                  HttpRequest httpRequest,
+                                                  HttpChannel httpChannel) {
         Map<String, String> params = Collections.emptyMap();
         HttpRequest httpRequestWithoutContentType = httpRequest.removeHeader("Content-Type");
 
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 0a14c3628146d..492ac452f65af 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -41,14 +41,19 @@ public class CompatibleHeaderCombinationTests extends ESTestCase {
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
     int OBSOLETE_VERSION = Version.CURRENT.major - 2;
 
-//    public void testX(){
-//
-//        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-//            expect(requestCreated(), isCompatible()));
-////        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-////            expect(requestCreated(), isCompatible()));
-//
-//    }
+
+    public void testCombinationsNotSure(){
+        // body not present, so it should either be current or error (mismatch version) - but fails as it considers it compatible
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible())));
+
+        //this should be an exception - content is not pvodied
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+    }
+
     public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(requestCreated(), isCompatible()));
@@ -65,8 +70,8 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible()))); // no body - content-type is ignored
+//        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
+//            expect(requestCreated(), not(isCompatible()))); // no body - content-type is ignored
 
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
             expect(requestCreated(), not(isCompatible())));
@@ -75,18 +80,18 @@ public void testAcceptAndContentTypeCombinations() {
             expect(requestCreated(), not(isCompatible())));
 
         //tests when body present and one of the headers missing
-        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+//        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent(),
+//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+//        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
+//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
         // with body present and contentType versioned - default the accept header version to the one from content-type
         createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
             expect(requestCreated(), not(isCompatible())));
 
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-            expect(requestCreated(), isCompatible()));
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
         //tests when body NOT present and one of the headers missing
         createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent(),
@@ -95,9 +100,9 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
 
-        //body not present - ignoring content-type. Accept header = null defaults to JSON and current version (non compatible)
+        //body not present - if any of headers has a version it will be used
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
+            expect(requestCreated(), isCompatible()));
         createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader(null), contentTypeHeader(null), bodyNotPresent(),
@@ -108,6 +113,8 @@ public void testAcceptAndContentTypeCombinations() {
             expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/json"), bodyPresent(),
             expect(requestCreated(), not(isCompatible())));
+        createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyPresent(),
+            expect(requestCreated(), not(isCompatible())));
     }
 
     public void testObsoleteVersion() {
@@ -118,9 +125,11 @@ public void testObsoleteVersion() {
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
     }
 
+
     public void testMediaTypeCombinations() {
+        //body not present - ignore content-type
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
+            expect(requestCreated(), isCompatible()));
         createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader("*/*"), contentTypeHeader("application/json"), bodyNotPresent(),
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
index 2f2f5fb76bfe7..093b5ce28651b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
@@ -219,5 +219,18 @@ public FakeRestRequest build() {
             FakeHttpRequest fakeHttpRequest = new FakeHttpRequest(method, path, content, headers);
             return new FakeRestRequest(xContentRegistry, fakeHttpRequest, params, new FakeHttpChannel(address));
         }
+
+        @Override
+        public String toString() {
+            return "Builder{" +
+                "xContentRegistry=" + xContentRegistry +
+                ", headers=" + headers +
+                ", params=" + params +
+                ", content=" + content +
+                ", path='" + path + '\'' +
+                ", method=" + method +
+                ", address=" + address +
+                '}';
+        }
     }
 }

From c96bd86797902623faeabf3e073497411d8cbd33 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 15 Apr 2020 10:40:37 +0200
Subject: [PATCH 49/51] fix netty test

---
 .../common/xcontent/XContentType.java         |   2 +-
 .../rest/Netty4BadRequestIT.java              |   6 +-
 .../org/elasticsearch/rest/RestRequest.java   | 103 ++++--------------
 .../CompatibleHeaderCombinationTests.java     |  28 ++---
 4 files changed, 36 insertions(+), 103 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index 8846cbeb1daa2..42129e4f5e4bb 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -172,7 +172,7 @@ public static String parseMediaType(String mediaType) {
         if (mediaType != null) {
             Matcher matcher = COMPATIBLE_API_HEADER_PATTERN.matcher(mediaType);
             if (matcher.find()) {
-                return matcher.group(1) + "/" + matcher.group(3).toLowerCase(Locale.ROOT);
+                return (matcher.group(1) + "/" + matcher.group(3)).toLowerCase(Locale.ROOT);
             }
         }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
index cc40163ea6ad0..ecf7e86e748ef 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
@@ -116,10 +116,12 @@ public void testInvalidHeaderCombinations() throws IOException {
         final ObjectPath objectPath = ObjectPath.createFromResponse(response);
         final Map<String, Object> map = objectPath.evaluate("error");
         assertThat(map.get("type"), equalTo("compatible_api_headers_combination_exception"));
-        assertThat(map.get("reason"), equalTo("Request with a body and incompatible Accept and Content-Type header values. " +
+        assertThat(map.get("reason"), equalTo("Content-Type and Accept headers have to match when content is present. " +
             "Accept=application/vnd.elasticsearch+json;compatible-with=8 " +
             "Content-Type=application/vnd.elasticsearch+json;compatible-with=7 " +
             "hasContent=true " +
-            "/_cluster/settings {} GET"));
+            "path=/_cluster/settings " +
+            "params={} " +
+            "method=GET"));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index d32e27164e93c..cd52770fd6683 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -146,8 +146,6 @@ public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRe
 
     private void addCompatibleParameter(boolean headersValidation) {
         if(headersValidation){
-//            Version compatible = compatibleWithVersion();
-//            if (Version.minimumRestCompatibilityVersion().equals(compatible)) {
             if(isRequestingCompatibility()) {
                 params().put(CompatibleConstants.COMPATIBLE_PARAMS_KEY,
                     String.valueOf(Version.minimumRestCompatibilityVersion().major));
@@ -165,104 +163,37 @@ private boolean isRequestingCompatibility() {
         byte contentTypeVersion = cVersion == null ? Version.CURRENT.major : Integer.valueOf(cVersion).byteValue();
 
         if(Version.CURRENT.major < acceptVersion || Version.CURRENT.major - acceptVersion > 1 ){
-//            throw new IllegalStateException("something about only 1 major version support");
             throw new CompatibleApiHeadersCombinationException(
-                String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                        "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                String.format(Locale.ROOT, "Unsupported version provided. " +
+                        "Accept=%s Content-Type=%s hasContent=%b path=%s params=%s method=%s", acceptHeader,
                     contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
         }
         if (hasContent()) {
             if(Version.CURRENT.major < contentTypeVersion || Version.CURRENT.major - contentTypeVersion > 1 ){
-//                throw new IllegalStateException("something about only 1 major version support");
                 throw new CompatibleApiHeadersCombinationException(
-                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                    String.format(Locale.ROOT, "Unsupported version provided. " +
+                            "Accept=%s Content-Type=%s hasContent=%b path=%s params=%s method=%s", acceptHeader,
                         contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
             }
 
             if (contentTypeVersion != acceptVersion) {
-//                throw new IllegalStateException("something about needing to match");
                 throw new CompatibleApiHeadersCombinationException(
-                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                    String.format(Locale.ROOT, "Content-Type and Accept headers have to match when content is present. " +
+                            "Accept=%s Content-Type=%s hasContent=%b path=%s params=%s method=%s", acceptHeader,
                         contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
             }
-        }
-
-        return contentTypeVersion < Version.CURRENT.major || acceptVersion < Version.CURRENT.major;
-    }
-
-    private Version compatibleWithVersion() {
-        String currentVersion = String.valueOf(Version.CURRENT.major);
-        String previousVersion = String.valueOf(Version.CURRENT.major - 1);
-
-        String acceptHeader = header(CompatibleConstants.COMPATIBLE_ACCEPT_HEADER);
-        String acceptVersion = XContentType.parseVersion(acceptHeader);
-        String contentTypeHeader = header(CompatibleConstants.COMPATIBLE_CONTENT_TYPE_HEADER);
-        String contentTypeVersion = XContentType.parseVersion(contentTypeHeader);
-
-        //TODO not sure about this one as this does not cover text formats
-        boolean isSupportedMediaTypeAccept = acceptHeader == null || XContentType.parseMediaType(acceptHeader) != null;
-
-        boolean isSupportedMediaTypeContentType = contentTypeHeader == null || XContentType.parseMediaType(contentTypeHeader) != null;
-
-        if (hasContent()) {
-            //both headers versioned
-            if (acceptVersion != null && contentTypeVersion != null) {
-                // both Accept and Content-Type are versioned and set to a previous version
-                if (previousVersion.equals(acceptVersion) && previousVersion.equals(contentTypeVersion)) {
-                    return Version.minimumRestCompatibilityVersion();
-                }
-                // both Accept and Content-Type are versioned to a current version
-                if (currentVersion.equals(acceptVersion) && currentVersion.equals(contentTypeVersion)) {
-                    return Version.CURRENT;
-                }
-                // both headers are versioned but set to incorrect version
-                throw new CompatibleApiHeadersCombinationException(
-                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
-                        contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
-            }
-            // Content-Type is versioned but accept is not present
-            if(previousVersion.equals(contentTypeVersion)
-                && (acceptHeader == null || acceptHeader.equals("*/*") )) {
-                return Version.minimumRestCompatibilityVersion();
-            }
-            // Content type is set (not versioned) but accept is not present. It will be defaulted to JSON
-            if(isSupportedMediaTypeContentType &&
-                (acceptHeader == null || acceptHeader.equals("*/*") )){//TODO when do we default this?
-                return Version.CURRENT;
-            }
-            // both Accept and Content-Type are not a compatible format, but are supported and not empty
-            if (isSupportedMediaTypeContentType && isSupportedMediaTypeAccept &&
-                acceptHeader != null && contentTypeHeader != null) {
-                return Version.CURRENT;
-            }
-        } else {
-            if (acceptVersion != null) {
-                //Accept header is versioned and set to previous
-                if (previousVersion.equals(acceptVersion)) {
-                    return Version.minimumRestCompatibilityVersion();
-                }
-                if (currentVersion.equals(acceptVersion)) {
-                    return Version.CURRENT;
-                }
-                // Accept header is versioned but set to incorrect version
+            // both headers should be versioned or none
+            if ((cVersion == null && aVersion!=null) || (aVersion ==null && cVersion!=null) ){
                 throw new CompatibleApiHeadersCombinationException(
-                    String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                            "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
+                    String.format(Locale.ROOT, "Versioning is required on both Content-Type and Accept headers. " +
+                            "Accept=%s Content-Type=%s hasContent=%b path=%s params=%s method=%s", acceptHeader,
                         contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
             }
-            //Accept header is not versioned but is supported
-            if (isSupportedMediaTypeAccept) {
-                return Version.CURRENT;
-            }
+
+            return contentTypeVersion < Version.CURRENT.major;
         }
 
-        throw new CompatibleApiHeadersCombinationException(
-            String.format(Locale.ROOT, "Request with a body and incompatible Accept and Content-Type header values. " +
-                    "Accept=%s Content-Type=%s hasContent=%b %s %s %s", acceptHeader,
-                contentTypeHeader, hasContent(), path(), params.toString(), method().toString()));
+        return acceptVersion < Version.CURRENT.major;
     }
 
     private static Map<String, String> params(final String uri) {
@@ -313,7 +244,13 @@ public static RestRequest requestWithoutContentType(NamedXContentRegistry xConte
             requestIdGenerator.incrementAndGet(), false);
     }
 
-
+    /**
+     * creates a Rest request when it is not able to pass a validation but a response is needed to be returned.
+     * @param xContentRegistry the content registry
+     * @param httpRequest      the http request
+     * @param httpChannel      the http channel
+     * @return a RestRequest without headers and parameters
+     */
     public static RestRequest requestNoValidation(NamedXContentRegistry xContentRegistry,
                                                   HttpRequest httpRequest,
                                                   HttpChannel httpChannel) {
diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index 492ac452f65af..f8becf5c83a19 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -41,13 +41,8 @@ public class CompatibleHeaderCombinationTests extends ESTestCase {
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
     int OBSOLETE_VERSION = Version.CURRENT.major - 2;
 
-
     public void testCombinationsNotSure(){
-        // body not present, so it should either be current or error (mismatch version) - but fails as it considers it compatible
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-
-        //this should be an exception - content is not pvodied
+        //this should be an exception - content-type is not provided
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
@@ -70,8 +65,8 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-//        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-//            expect(requestCreated(), not(isCompatible()))); // no body - content-type is ignored
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
+            expect(requestCreated(), not(isCompatible()))); // no body - content-type is ignored
 
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
             expect(requestCreated(), not(isCompatible())));
@@ -79,16 +74,15 @@ public void testAcceptAndContentTypeCombinations() {
         createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
 
-        //tests when body present and one of the headers missing
-//        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent(),
-//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        //tests when body present and one of the headers missing - versioning is required on both when body is present
+        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-//        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
-//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
-        // with body present and contentType versioned - default the accept header version to the one from content-type
         createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
+            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
 
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
             expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
@@ -102,7 +96,7 @@ public void testAcceptAndContentTypeCombinations() {
 
         //body not present - if any of headers has a version it will be used
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), isCompatible()));
+            expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader(null), contentTypeHeader(null), bodyNotPresent(),
@@ -129,7 +123,7 @@ public void testObsoleteVersion() {
     public void testMediaTypeCombinations() {
         //body not present - ignore content-type
         createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), isCompatible()));
+            expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent(),
             expect(requestCreated(), not(isCompatible())));
         createRequestWith(acceptHeader("*/*"), contentTypeHeader("application/json"), bodyNotPresent(),

From 10e8d1bf5ddaf664927d22e3d27de779a3fc46c5 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Wed, 15 Apr 2020 17:49:07 +0200
Subject: [PATCH 50/51] refactor test to be using hamcrest style

---
 .../CompatibleHeaderCombinationTests.java     | 215 ++++++++----------
 1 file changed, 96 insertions(+), 119 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
index f8becf5c83a19..e38b7f9efab38 100644
--- a/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHeaderCombinationTests.java
@@ -33,167 +33,151 @@
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 
 public class CompatibleHeaderCombinationTests extends ESTestCase {
     int CURRENT_VERSION = Version.CURRENT.major;
     int PREVIOUS_VERSION = Version.CURRENT.major - 1;
     int OBSOLETE_VERSION = Version.CURRENT.major - 2;
 
-    public void testCombinationsNotSure(){
-        //this should be an exception - content-type is not provided
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-    }
-
     public void testAcceptAndContentTypeCombinations() {
-        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-            expect(requestCreated(), isCompatible()));
+        assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
 
-        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), isCompatible()));
+        assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
 
-        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent()));
 
-        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
-            expect(requestCreated(), isCompatible())); // no body - content-type is ignored
+        // no body - content-type is ignored
+        assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
+        // no body - content-type is ignored
+        assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()));
 
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible()))); // no body - content-type is ignored
+        assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
-
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
+        assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
         //tests when body present and one of the headers missing - versioning is required on both when body is present
-        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent()));
 
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent()));
 
-        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent()));
 
-        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()));
 
         //tests when body NOT present and one of the headers missing
-        createRequestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent(),
-            expect(requestCreated(), isCompatible()));
+        assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
+
+        assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
-        createRequestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
+        //body not present - accept header is missing - it will default to Current version. Version on content type is ignored
+        assertThat(requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
-        //body not present - if any of headers has a version it will be used
-        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader(null), contentTypeHeader(null), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
+        assertThat(requestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
+
+        assertThat(requestWith(acceptHeader(null), contentTypeHeader(null), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
         //Accept header = application/json means current version. If body is provided then accept and content-Type should be the same
-        createRequestWith(acceptHeader("application/json"), contentTypeHeader(null), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/json"), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
+        assertThat(requestWith(acceptHeader("application/json"), contentTypeHeader(null), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
+
+        assertThat(requestWith(acceptHeader("application/json"), contentTypeHeader("application/json"), bodyPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
+
+        assertThat(requestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
     }
 
     public void testObsoleteVersion() {
-        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent()));
 
-        createRequestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(null), bodyNotPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(null), bodyNotPresent()));
     }
 
 
     public void testMediaTypeCombinations() {
         //body not present - ignore content-type
-        createRequestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader("*/*"), contentTypeHeader("application/json"), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
+        assertThat(requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
+
+        assertThat(requestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
+
+        assertThat(requestWith(acceptHeader("*/*"), contentTypeHeader("application/json"), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
         //this is for instance used by SQL
-        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/cbor"), bodyPresent(),
-            expect(requestCreated(), not(isCompatible())));
+        assertThat(requestWith(acceptHeader("application/json"), contentTypeHeader("application/cbor"), bodyPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
-        createRequestWith(acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"),
-            contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=7"), bodyPresent(),
-            expect(requestCreated(), isCompatible()));
+        assertThat(requestWith(acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"),
+            contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=7"), bodyPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
 
         //different versions on different media types
-        createRequestWith(acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"),
-            contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=8"), bodyPresent(),
-            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
+        expectThrows(RestRequest.CompatibleApiHeadersCombinationException.class, () ->
+            requestWith(acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"),
+                contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=8"), bodyPresent()));
     }
 
-    public void testTextMediaTypes(){
-        createRequestWith(acceptHeader("text/tab-separated-values"), contentTypeHeader("application/json"), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader("text/plain"), contentTypeHeader("application/json"), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
-        createRequestWith(acceptHeader("text/csv"), contentTypeHeader("application/json"), bodyNotPresent(),
-            expect(requestCreated(), not(isCompatible())));
+    public void testTextMediaTypes() {
+        assertThat(requestWith(acceptHeader("text/tab-separated-values"), contentTypeHeader("application/json"), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
+
+        assertThat(requestWith(acceptHeader("text/plain"), contentTypeHeader("application/json"), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
+
+        assertThat(requestWith(acceptHeader("text/csv"), contentTypeHeader("application/json"), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), not(isCompatible())));
 
         //versioned
-        createRequestWith(acceptHeader("text/vnd.elasticsearch+tab-separated-values;compatible-with=7"),
-            contentTypeHeader(7), bodyNotPresent(),
-            expect(requestCreated(), isCompatible()));
-        createRequestWith(acceptHeader("text/vnd.elasticsearch+plain;compatible-with=7"),
-            contentTypeHeader(7), bodyNotPresent(),
-            expect(requestCreated(), isCompatible()));
-        createRequestWith(acceptHeader("text/vnd.elasticsearch+csv;compatible-with=7"),
-            contentTypeHeader(7), bodyNotPresent(),
-            expect(requestCreated(), isCompatible()));
-    }
-//    public void testMalfunctionedMediaTypes(){
-//        createRequestWith(acceptHeader("application/json"), contentTypeHeader("application/something+json"), bodyPresent(),
-//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-//        createRequestWith(acceptHeader("application/something+json"), contentTypeHeader("application/json"), bodyNotPresent(),
-//            expect(exceptionDuringCreation(RestRequest.CompatibleApiHeadersCombinationException.class)));
-//    }
-
-    private Matcher<FakeRestRequest.Builder> exceptionDuringCreation(Class<? extends Exception> exceptionClass) {
-        return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {
-            try {
-                builder.build();
-            } catch (Exception e) {
-                return e;
-            }
-            return null;
-        }, instanceOf(exceptionClass));
+        assertThat(requestWith(acceptHeader("text/vnd.elasticsearch+tab-separated-values;compatible-with=7"),
+            contentTypeHeader(7), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
 
+        assertThat(requestWith(acceptHeader("text/vnd.elasticsearch+plain;compatible-with=7"),
+            contentTypeHeader(7), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
+
+        assertThat(requestWith(acceptHeader("text/vnd.elasticsearch+csv;compatible-with=7"),
+            contentTypeHeader(7), bodyNotPresent()),
+            Matchers.allOf(requestCreated(), isCompatible()));
     }
 
-    private Matcher<FakeRestRequest.Builder> requestCreated() {
-        return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> builder.build(), Matchers.notNullValue());
+    private Matcher<RestRequest> requestCreated() {
+        return Matchers.not(nullValue(RestRequest.class));
     }
 
-    private Matcher<FakeRestRequest.Builder> isCompatible() {
+    private Matcher<RestRequest> isCompatible() {
         return requestHasVersion(Version.CURRENT.major - 1);
     }
 
-    private Matcher<FakeRestRequest.Builder> requestHasVersion(int version) {
-        return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(builder -> {
-            FakeRestRequest build = builder.build();
-            return build.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY); //TODO to be refactored into getVersion
-        }, equalTo(String.valueOf(version)));
+    private Matcher<RestRequest> requestHasVersion(int version) {
+        return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(build ->
+                build.param(CompatibleConstants.COMPATIBLE_PARAMS_KEY) //TODO to be refactored into getVersion
+            , equalTo(String.valueOf(version)));
     }
 
     private String bodyNotPresent() {
@@ -234,14 +218,7 @@ private List<String> mediaType(Integer version) {
         return null;
     }
 
-    private Matcher<FakeRestRequest.Builder> expect(Matcher<FakeRestRequest.Builder>... matchers) {
-        return Matchers.allOf(matchers);
-    }
-
-    private void createRequestWith(List<String> accept,
-                                   List<String> contentType,
-                                   String body,
-                                   Matcher<FakeRestRequest.Builder> matcher) {
+    private RestRequest requestWith(List<String> accept, List<String> contentType, String body) {
         FakeRestRequest.Builder builder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
 
         builder.withHeaders(createHeaders(accept, contentType));
@@ -249,7 +226,7 @@ private void createRequestWith(List<String> accept,
             // xContentType header is set explicitly in headers
             builder.withContent(new BytesArray(body), null);
         }
-        assertThat(builder, matcher);
+        return builder.build();
     }
 
     private Map<String, List<String>> createHeaders(List<String> accept, List<String> contentType) {

From 3181daf8323a0cda918324aa98a67b426a75d7ad Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 16 Apr 2020 11:44:43 +0200
Subject: [PATCH 51/51] cleanup

---
 .../org/elasticsearch/gradle/http/WaitForHttpResource.java    | 3 ---
 server/src/main/java/org/elasticsearch/Version.java           | 3 +--
 .../main/java/org/elasticsearch/rest/CompatibleConstants.java | 2 +-
 .../elasticsearch/test/rest/yaml/ClientYamlTestClient.java    | 4 +++-
 4 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java b/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
index d097fabbbbced..db805b26cdca5 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
@@ -130,8 +130,6 @@ public boolean wait(int durationInMs) throws GeneralSecurityException, Interrupt
 
     protected void checkResource(SSLContext ssl) throws IOException {
         final HttpURLConnection connection = buildConnection(ssl);
-        connection.setRequestProperty("Accept", "application/json");
-
         connection.connect();
         final Integer response = connection.getResponseCode();
         if (validResponseCodes.contains(response)) {
@@ -146,7 +144,6 @@ HttpURLConnection buildConnection(SSLContext ssl) throws IOException {
         final HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
         configureSslContext(connection, ssl);
         configureBasicAuth(connection);
-
         connection.setRequestMethod("GET");
         return connection;
     }
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index d1e004aecd90a..7b607c2550de0 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -89,7 +89,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
         for (final Field declaredField : Version.class.getFields()) {
             if (declaredField.getType().equals(Version.class)) {
                 final String fieldName = declaredField.getName();
-                if (fieldName.equals("CURRENT") || fieldName.equals("V_EMPTY") || fieldName.equals("PREVIOUS")) {
+                if (fieldName.equals("CURRENT") || fieldName.equals("V_EMPTY")) {
                     continue;
                 }
                 assert fieldName.matches("V_\\d+_\\d+_\\d+")
@@ -406,7 +406,6 @@ public static List<Version> getDeclaredVersions(final Class<?> versionClass) {
             }
             switch (field.getName()) {
                 case "CURRENT":
-                case "PREVIOUS":
                 case "V_EMPTY":
                     continue;
             }
diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
index 14c10345a23b6..2f3f13b893fe3 100644
--- a/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
+++ b/server/src/main/java/org/elasticsearch/rest/CompatibleConstants.java
@@ -29,6 +29,6 @@ public class CompatibleConstants {
     public static final String COMPATIBLE_ACCEPT_HEADER = "Accept";
     public static final String COMPATIBLE_CONTENT_TYPE_HEADER = "Content-Type";
     public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With";
-    public static final String COMPATIBLE_VERSION = String.valueOf(Version.CURRENT.major - 1);
+    public static final String COMPATIBLE_VERSION = String.valueOf(Version.minimumRestCompatibilityVersion().major);
 
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
index c0d589fb2d9a3..0040116ff4fce 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
@@ -180,7 +180,9 @@ public ClientYamlTestResponse callApi(String apiName, Map<String, String> params
         for (Map.Entry<String, String> param : queryStringParams.entrySet()) {
             request.addParameter(param.getKey(), param.getValue());
         }
-        if(entity!=null && entity.getContentType() != null){
+        // that means content type was set based on Content-Type from entity.
+        // Setting content-type on both entity and headers will result in exception in server
+        if (entity != null && entity.getContentType() != null) {
             headers.remove(HTTP.CONTENT_TYPE);
         }
         request.setEntity(entity);